karafka 1.2.13 → 1.3.0.rc1

metadata.gz.sig

@@ -0,0 +1,4 @@
+4�ȭM�*�%�T�d��ڇ�PO�C!�p����-\�Uxu`��GU��<4���_�/\)N�$�]'YPu]�c��5�Պ�k�Q��$���N_/��,�Ey�T1�������.���DZ�٨�޼����$�3�������}zF�+y"FF����� :)?Q��Ȇ=�
+�\DD<��n�UPLJ|����n��퐨Ѥ���t`��\b@�
+�1�������3)y��X����<�KGq�[Y��
+��#x߳4�L=j��R�Kp��CN}�{�2��~Yelf�VW4�p���6DJ`��@��}�;�ګY��+(`x�i/�[�����RhaS*�"��9
Rz*���"�Q9�A��;f�d؏c��}�������X�w��-�

data/lib/karafka/callbacks.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Additional callbacks that are used to trigger some things in given places during the
-  # system lifecycle
-  # @note Those callbacks aren't the same as consumer callbacks as they are not related to the
-  #   lifecycle of particular messages fetches but rather to the internal flow process.
-  #   They cannot be defined on a consumer callback level because for some of those,
-  #   there aren't consumers in the memory yet and/or they aren't per consumer thread
-  module Callbacks
-    # Types of system callbacks that we have that are not related to consumers
-    TYPES = %i[
-      after_init
-      before_fetch_loop
-    ].freeze
-
-    class << self
-      TYPES.each do |callback_type|
-        # Executes given callbacks set at a given moment with provided arguments
-        define_method callback_type do |*args|
-          Karafka::App
-            .config
-            .callbacks
-            .send(callback_type)
-            .each { |callback| callback.call(*args) }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/callbacks/config.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Callbacks
-    # Additional configuration required to store procs that we will execute upon callback trigger
-    module Config
-      # Builds up internal callback accumulators
-      # @param klass [Class] Class that we extend with callback config
-      def self.extended(klass)
-        # option internal [Hash] - optional - internal karafka configuration settings that should
-        #   never be changed by users directly
-        klass.setting :callbacks do
-          Callbacks::TYPES.each do |callback_type|
-            # option [Array<Proc>] an array of blocks that will be executed at a given moment
-            #   depending on the callback type
-            setting callback_type, []
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/callbacks/dsl.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Callbacks
-    # App level dsl to define callbacks
-    module Dsl
-      Callbacks::TYPES.each do |callback_type|
-        # Allows us to define a block, that will be executed for a given moment
-        # @param [Block] block that should be executed after the initialization process
-        define_method callback_type do |&block|
-          config.callbacks.send(callback_type).push block
-        end
-      end
-    end
-  end
-end

data/lib/karafka/connection/delegator.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Class that delegates processing of messages for which we listen to a proper processor
-    module Delegator
-      class << self
-        # Delegates messages (does something with them)
-        # It will either schedule or run a proper processor action for messages
-        # @note This should be looped to obtain a constant delegating of new messages
-        # @note We catch all the errors here, to make sure that none failures
-        #   for a given consumption will affect other consumed messages
-        #   If we wouldn't catch it, it would propagate up until killing the thread
-        # @note It is a one huge method, because of performance reasons. It is much faster then
-        #   using send or invoking additional methods
-        # @param group_id [String] group_id of a group from which a given message came
-        # @param kafka_messages [Array<Kafka::FetchedMessage>] raw messages fetched from kafka
-        def call(group_id, kafka_messages)
-          # @note We always get messages by topic and partition so we can take topic from the
-          # first one and it will be valid for all the messages
-          topic = Persistence::Topic.fetch(group_id, kafka_messages[0].topic)
-          consumer = Persistence::Consumer.fetch(topic, kafka_messages[0].partition)
-
-          Karafka.monitor.instrument(
-            'connection.delegator.call',
-            caller: self,
-            consumer: consumer,
-            kafka_messages: kafka_messages
-          ) do
-            # Depending on a case (persisted or not) we might use new consumer instance per
-            # each batch, or use the same one for all of them (for implementing buffering, etc.)
-            if topic.batch_consuming
-              consumer.params_batch = kafka_messages
-              consumer.call
-            else
-              kafka_messages.each do |kafka_message|
-                consumer.params_batch = [kafka_message]
-                consumer.call
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/listener.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Default listener that hooks up to our instrumentation and uses its events for logging
-    # It can be removed/replaced or anything without any harm to the Karafka app flow
-    module Listener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        debug
-        info
-        error
-        fatal
-      ].freeze
-
-      # Injects WaterDrop listener logger actions
-      extend WaterDrop::Instrumentation::Listener
-
-      class << self
-        # Logs details about incoming messages and with which consumer we will consume them
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_delegator_call(event)
-          consumer = event[:consumer]
-          topic = consumer.topic.name
-          kafka_messages = event[:kafka_messages]
-          info "#{kafka_messages.count} messages on #{topic} topic delegated to #{consumer.class}"
-        end
-
-        # Logs details about each received message value parsing
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse(event)
-          # Keep in mind, that a caller here is a param object not a controller,
-          # so it returns a topic as a string, not a routing topic
-          debug "Params parsing for #{event[:caller].topic} topic successful in #{event[:time]} ms"
-        end
-
-        # Logs unsuccessful parsing attempts of incoming data
-        # @param event [Dry::Events::Event] event details including payload
-        def on_params_params_parse_error(event)
-          error "Params parsing error for #{event[:caller].topic} topic: #{event[:error]}"
-        end
-
-        # Logs errors that occured in a listener fetch loop
-        # @param event [Dry::Events::Event] event details including payload
-        # @note It's an error as we can recover from it not a fatal
-        def on_connection_listener_fetch_loop_error(event)
-          error "Listener fetch loop error: #{event[:error]}"
-        end
-
-        # Logs errors that are related to the connection itself
-        # @note Karafka will attempt to reconnect, so an error not a fatal
-        # @param event [Dry::Events::Event] event details including payload
-        def on_connection_client_fetch_loop_error(event)
-          error "Client fetch loop error: #{event[:error]}"
-        end
-
-        # Logs info about crashed fetcher
-        # @note If this happens, Karafka will shutdown as it means a critical error
-        #   in one of the threads
-        # @param event [Dry::Events::Event] event details including payload
-        def on_fetcher_call_error(event)
-          fatal "Fetcher crash due to an error: #{event[:error]}"
-        end
-
-        # Logs info about processing of a certain dataset with an inline backend
-        # @param event [Dry::Events::Event] event details including payload
-        def on_backends_inline_process(event)
-          count = event[:caller].send(:params_batch).to_a.size
-          topic = event[:caller].topic.name
-          time = event[:time]
-          info "Inline processing of topic #{topic} with #{count} messages took #{time} ms"
-        end
-
-        # Logs info about system signals that Karafka received
-        # @param event [Dry::Events::Event] event details including payload
-        def on_process_notice_signal(event)
-          info "Received #{event[:signal]} system signal"
-        end
-
-        # Logs info about responder usage withing a controller flow
-        # @param event [Dry::Events::Event] event details including payload
-        def on_consumers_responders_respond_with(event)
-          calling = event[:caller].class
-          responder = calling.topic.responder
-          data = event[:data]
-          info "Responded from #{calling} using #{responder} with following data #{data}"
-        end
-
-        # Logs info that we're going to stop the Karafka server
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { info "Stopping Karafka server #{::Process.pid}" }
-        end
-
-        # Logs an error that Karafka was unable to stop the server gracefully and it had to do a
-        #   forced exit
-        # @param _event [Dry::Events::Event] event details including payload
-        def on_server_stop_error(_event)
-          # We use a separate thread as logging can't be called from trap context
-          Thread.new { error "Forceful Karafka server #{::Process.pid} stop" }
-        end
-
-        USED_LOG_LEVELS.each do |log_level|
-          define_method log_level do |*args|
-            Karafka.logger.send(log_level, *args)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/loader.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Loader for requiring all the files in a proper order
-  module Loader
-    # Order in which we want to load app files
-    DIRS = %w[
-      lib
-      app
-    ].freeze
-
-    # Will load files in a proper order (based on DIRS)
-    # @param [String] root path from which we want to start
-    def self.load(root)
-      DIRS.each do |dir|
-        path = File.join(root, dir)
-        next unless File.exist?(path)
-        load!(path)
-      end
-    end
-
-    # Requires all the ruby files from one path in a proper order
-    # @param path [String] path (dir) from which we want to load ruby files in a proper order
-    def self.load!(path)
-      require_all(path)
-    end
-  end
-end

data/lib/karafka/params/dsl.rb

@@ -1,158 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Params namespace encapsulating all the logic that is directly related to params handling
-  module Params
-    # Dsl for Karafka params. We don't provide the params class here as we want to allow users to
-    # use either hash (default) or Rails hash with indifferent access as a base for their params
-    #
-    # We do that because both of them have their own advantages and we don't want to enforce users
-    # to handle things differently if they already use any of those
-    #
-    # It provides lazy loading not only until the first usage, but also allows us to skip
-    # using parser until we execute our logic. That way we can operate with
-    # heavy-parsing data without slowing down the whole application.
-    module Dsl
-      # Params keys that are "our" and internal. We use this list for additional backends
-      # that somehow operatae on those keys
-      SYSTEM_KEYS = %w[
-        parser
-        value
-        partition
-        offset
-        key
-        create_time
-        receive_time
-        topic
-        parsed
-      ].freeze
-
-      # Params attributes that should be available via a method call invocation for Kafka
-      # client compatibility.
-      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
-      # uses those fields via method calls, so in order to be able to pass there our params
-      # objects, have to have same api.
-      METHOD_ATTRIBUTES = %w[
-        topic
-        partition
-        offset
-        key
-        create_time
-        receive_time
-      ].freeze
-
-      private_constant :METHOD_ATTRIBUTES
-
-      # Class methods required by params to work
-      module ClassMethods
-        # We allow building instances only via the #build method
-
-        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
-        #   in case of building params inside main Karafka process in
-        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
-        # @param parser [Class] parser class that we will use to unparse data
-        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
-        #   retrieving data that we've got from Kafka
-        # @example Build params instance from a hash
-        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
-        # @example Build params instance from a Kafka::FetchedMessage object
-        #   Karafka::Params::Params.build(message) #=> params object
-        def build(message, parser)
-          instance = new
-          instance['parser'] = parser
-
-          # Non kafka fetched message can happen when we interchange data with an
-          # additional backend
-          if message.is_a?(Kafka::FetchedMessage)
-            instance.send(
-              :merge!,
-              'value' => message.value,
-              'partition' => message.partition,
-              'offset' => message.offset,
-              'key' => message.key,
-              'create_time' => message.create_time,
-              'receive_time' => Time.now,
-              # When we get raw messages, they might have a topic, that was modified by a
-              # topic mapper. We need to "reverse" this change and map back to the non-modified
-              # format, so our internal flow is not corrupted with the mapping
-              'topic' => Karafka::App.config.topic_mapper.incoming(message.topic)
-            )
-          else
-            instance.send(:merge!, message)
-          end
-
-          instance
-        end
-      end
-
-      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
-      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
-      #   to the current object. This object will be also marked as already parsed, so we won't
-      #   parse it again.
-      def retrieve!
-        return self if self['parsed']
-        self['parsed'] = true
-
-        parsed_data = parse(self['value'])
-        delete('value')
-        merge!(parsed_data)
-      end
-
-      # Includes and extends the base params klass with everything that is needed by Karafka to
-      #   fully work in any conditions.
-      # @param params_klass [Karafka::Params::Params] initialized params class that we will
-      #   use for a given Karafka process
-      def self.included(params_klass)
-        params_klass.extend(Dsl::ClassMethods)
-
-        METHOD_ATTRIBUTES.each do |attr|
-          # Defines a method call accessor to a particular hash field.
-          # @note Won't work for complex key names that contain spaces, etc
-          # @param key [Symbol] name of a field that we want to retrieve with a method call
-          # @example
-          #   key_attr_reader :example
-          #   params.example #=> 'my example value'
-          params_klass.send :define_method, attr do
-            self[attr]
-          end
-        end
-
-        params_klass.send :private, :merge!
-        params_klass.send :private, :parse
-      end
-
-      private
-
-      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
-      #  and in a other_hash - it will not replace keys that are the same in our hash
-      #  and in the other one. This protects some important Karafka params keys that cannot be
-      #  replaced with custom values from incoming Kafka message
-      # @param other_hash [Hash] hash that we want to merge into current
-      # @return [Karafka::Params::Params] our parameters hash with merged values
-      # @example Merge with hash without same keys
-      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
-      # @example Merge with hash with same keys (symbol based)
-      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (string based)
-      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
-      # @example Merge with hash with same keys (current string based)
-      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
-      def merge!(other_hash)
-        super(other_hash) { |_key, base_value, _new_value| base_value }
-      end
-
-      # @param value [String] Raw data that we want to parse using consumer parser
-      # @note If something goes wrong, it will return raw data in a hash with a message key
-      # @return [Hash] parsed data or a hash with message key containing raw data if something
-      #   went wrong during parsing
-      def parse(value)
-        Karafka.monitor.instrument('params.params.parse', caller: self) do
-          self['parser'].parse(value)
-        end
-      rescue ::Karafka::Errors::ParserError => e
-        Karafka.monitor.instrument('params.params.parse.error', caller: self, error: e)
-        raise e
-      end
-    end
-  end
-end

data/lib/karafka/parsers/json.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Module for all supported by default parsers for incoming/outgoing data
-  module Parsers
-    # Default Karafka Json parser for serializing and deserializing data
-    class Json
-      # @param content [String] content based on which we want to get our hash
-      # @return [Hash] hash with parsed JSON data
-      # @example
-      #   Json.parse("{\"a\":1}") #=> { 'a' => 1 }
-      def self.parse(content)
-        ::MultiJson.load(content)
-      rescue ::MultiJson::ParseError => e
-        raise ::Karafka::Errors::ParserError, e
-      end
-
-      # @param content [Object] any object that we want to convert to a json string
-      # @return [String] Valid JSON string containing serialized data
-      # @raise [Karafka::Errors::ParserError] raised when we don't have a way to parse
-      #   given content to a json string format
-      # @note When string is passed to this method, we assume that it is already a json
-      #   string and we don't serialize it again. This allows us to serialize data before
-      #   it is being forwarded to a parser if we want to have a custom (not that simple)
-      #   json serialization
-      #
-      # @example From an ActiveRecord object
-      #   Json.generate(Repository.first) #=> "{\"repository\":{\"id\":\"04b504e0\"}}"
-      # @example From a string (no changes)
-      #   Json.generate("{\"a\":1}") #=> "{\"a\":1}"
-      def self.generate(content)
-        return content if content.is_a?(String)
-        return content.to_json if content.respond_to?(:to_json)
-        raise Karafka::Errors::ParserError, content
-      end
-    end
-  end
-end