karafka 1.0.0 → 1.2.0

data/lib/karafka/loader.rb

@@ -5,7 +5,6 @@ module Karafka
   module Loader
     # Order in which we want to load app files
     DIRS = %w[
-      config/initializers
       lib
       app
     ].freeze

data/lib/karafka/params/dsl.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Params namespace encapsulating all the logic that is directly related to params handling
+  module Params
+    # Dsl for Karafka params. We don't provide the params class here as we want to allow users to
+    # use either hash (default) or Rails hash with indifferent access as a base for their params
+    #
+    # We do that because both of them have their own advantages and we don't want to enforce users
+    # to handle things differently if they already use any of those
+    #
+    # It provides lazy loading not only until the first usage, but also allows us to skip
+    # using parser until we execute our logic. That way we can operate with
+    # heavy-parsing data without slowing down the whole application.
+    module Dsl
+      # Params keys that are "our" and internal. We use this list for additional backends
+      # that somehow operatae on those keys
+      SYSTEM_KEYS = %w[
+        parser
+        value
+        partition
+        offset
+        key
+        create_time
+        receive_time
+        topic
+        parsed
+      ].freeze
+
+      # Params attributes that should be available via a method call invocation for Kafka
+      # client compatibility.
+      # Kafka passes internally Kafka::FetchedMessage object and the ruby-kafka consumer
+      # uses those fields via method calls, so in order to be able to pass there our params
+      # objects, have to have same api.
+      METHOD_ATTRIBUTES = %w[
+        topic
+        partition
+        offset
+        key
+        create_time
+        receive_time
+      ].freeze
+
+      private_constant :METHOD_ATTRIBUTES
+
+      # Class methods required by params to work
+      module ClassMethods
+        # We allow building instances only via the #build method
+
+        # @param message [Kafka::FetchedMessage, Hash] message that we get out of Kafka
+        #   in case of building params inside main Karafka process in
+        #   Karafka::Connection::Consumer, or a hash when we retrieve data that is already parsed
+        # @param parser [Class] parser class that we will use to unparse data
+        # @return [Karafka::Params::Params] Karafka params object not yet used parser for
+        #   retrieving data that we've got from Kafka
+        # @example Build params instance from a hash
+        #   Karafka::Params::Params.build({ key: 'value' }) #=> params object
+        # @example Build params instance from a Kafka::FetchedMessage object
+        #   Karafka::Params::Params.build(message) #=> params object
+        def build(message, parser)
+          instance = new
+          instance['parser'] = parser
+
+          # Non kafka fetched message can happen when we interchange data with an
+          # additional backend
+          if message.is_a?(Kafka::FetchedMessage)
+            instance.send(
+              :merge!,
+              'value' => message.value,
+              'partition' => message.partition,
+              'offset' => message.offset,
+              'key' => message.key,
+              'create_time' => message.create_time,
+              'receive_time' => Time.now,
+              # When we get raw messages, they might have a topic, that was modified by a
+              # topic mapper. We need to "reverse" this change and map back to the non-modified
+              # format, so our internal flow is not corrupted with the mapping
+              'topic' => Karafka::App.config.topic_mapper.incoming(message.topic)
+            )
+          else
+            instance.send(:merge!, message)
+          end
+
+          instance
+        end
+      end
+
+      # @return [Karafka::Params::Params] this will trigger parser execution. If we decide to
+      #   retrieve data, parser will be executed to parse data. Output of parsing will be merged
+      #   to the current object. This object will be also marked as already parsed, so we won't
+      #   parse it again.
+      def retrieve!
+        return self if self['parsed']
+        self['parsed'] = true
+
+        merge!(parse(delete('value')))
+      end
+
+      # Includes and extends the base params klass with everything that is needed by Karafka to
+      #   fully work in any conditions.
+      # @param params_klass [Karafka::Params::Params] initialized params class that we will
+      #   use for a given Karafka process
+      def self.included(params_klass)
+        params_klass.extend(Dsl::ClassMethods)
+
+        METHOD_ATTRIBUTES.each do |attr|
+          # Defines a method call accessor to a particular hash field.
+          # @note Won't work for complex key names that contain spaces, etc
+          # @param key [Symbol] name of a field that we want to retrieve with a method call
+          # @example
+          #   key_attr_reader :example
+          #   params.example #=> 'my example value'
+          params_klass.send :define_method, attr do
+            self[attr]
+          end
+        end
+
+        params_klass.send :private, :merge!
+        params_klass.send :private, :parse
+      end
+
+      private
+
+      # Overwritten merge! method - it behaves differently for keys that are the same in our hash
+      #  and in a other_hash - it will not replace keys that are the same in our hash
+      #  and in the other one. This protects some important Karafka params keys that cannot be
+      #  replaced with custom values from incoming Kafka message
+      # @param other_hash [Hash] hash that we want to merge into current
+      # @return [Karafka::Params::Params] our parameters hash with merged values
+      # @example Merge with hash without same keys
+      #   new(a: 1, b: 2).merge!(c: 3) #=> { a: 1, b: 2, c: 3 }
+      # @example Merge with hash with same keys (symbol based)
+      #   new(a: 1).merge!(a: 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (string based)
+      #   new(a: 1).merge!('a' => 2) #=> { a: 1 }
+      # @example Merge with hash with same keys (current string based)
+      #   new('a' => 1).merge!(a: 2) #=> { a: 1 }
+      def merge!(other_hash)
+        super(other_hash) { |_key, base_value, _new_value| base_value }
+      end
+
+      # @param value [String] Raw data that we want to parse using consumer parser
+      # @note If something goes wrong, it will return raw data in a hash with a message key
+      # @return [Hash] parsed data or a hash with message key containing raw data if something
+      #   went wrong during parsing
+      def parse(value)
+        Karafka.monitor.instrument('params.params.parse', caller: self) do
+          self['parser'].parse(value)
+        end
+      rescue ::Karafka::Errors::ParserError => e
+        Karafka.monitor.instrument('params.params.parse.error', caller: self, error: e)
+        raise e
+      end
+    end
+  end
+end

data/lib/karafka/params/params_batch.rb

@@ -4,7 +4,7 @@ module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
     # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_received that rejects some incoming messages without using params
+    #   process if we have after_fetch that rejects some incoming messages without using params
     #   It can be also used when handling really heavy data (in terms of parsing).
     class ParamsBatch
       include Enumerable
@@ -13,7 +13,7 @@ module Karafka
       # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
       # @param topic_parser [Class] topic parser for unparsing messages values
       def initialize(messages_batch, topic_parser)
-        @params_batch = messages_batch.map do |message|
+        @params_batch = messages_batch.map! do |message|
           Karafka::Params::Params.build(message, topic_parser)
         end
       end
@@ -32,6 +32,11 @@ module Karafka
         each(&:itself)
       end
 
+      # @return [Karafka::Params::Params] last element after the unparsing process
+      def last
+        @params_batch.last.retrieve!
+      end
+
       # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
       def to_a
         @params_batch

data/lib/karafka/patches/dry_configurable.rb

@@ -13,23 +13,23 @@ module Karafka
       def initialize(*args)
         super
 
-        @config.each do |key, _value|
-          rebuild(key)
-        end
+        @config.each_key(&method(:rebuild))
       end
 
       private
 
       # Method that rebuilds a given accessor, so when it consists a proc value, it will
-      # evaluate it upon return
+      # evaluate it upon return for blocks that don't require any arguments, otherwise
+      # it will return the block
       # @param method_name [Symbol] name of an accessor that we want to rebuild
       def rebuild(method_name)
         define_singleton_method method_name do
-          super().is_a?(Proc) ? super().call : super()
+          value = super()
+          return value unless value.is_a?(Proc)
+          return value unless value.parameters.empty?
+          value.call
         end
       end
     end
   end
 end
-
-::Dry::Configurable::Config.prepend(Karafka::Patches::DryConfigurable)

data/lib/karafka/patches/ruby_kafka.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    # Patches for Ruby Kafka gem
+    module RubyKafka
+      # This patch allows us to inject business logic in between fetches and before the consumer
+      # stop, so we can perform stop commit or anything else that we need since
+      # ruby-kafka fetch loop does not allow that directly
+      # We don't wan't to use poll ruby-kafka api as it brings many more problems that we would
+      # have to take care of. That way, nothing like that ever happens but we get the control
+      # over the stopping process that we need (since we're the once that initiate it for each
+      # thread)
+      def consumer_loop
+        super do
+          consumers = Karafka::Persistence::Consumer
+                      .all
+                      .values
+                      .flat_map(&:values)
+                      .select { |ctrl| ctrl.respond_to?(:run_callbacks) }
+
+          if Karafka::App.stopped?
+            consumers.each { |ctrl| ctrl.run_callbacks :before_stop }
+            Karafka::Persistence::Client.read.stop
+          else
+            consumers.each { |ctrl| ctrl.run_callbacks :before_poll }
+            yield
+            consumers.each { |ctrl| ctrl.run_callbacks :after_poll }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      # @param client [Karafka::Connection::Client] messages consumer client of
+      #   a current thread
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      def self.write(client)
+        Thread.current[PERSISTENCE_SCOPE] = client
+      end
+
+      # @return [Karafka::Connection::Client] persisted messages consumer client
+      # @raise [Karafka::Errors::MissingConsumer] raised when no thread messages consumer
+      #   client but we try to use it anyway
+      def self.read
+        Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClient)
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
+  module Persistence
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
+    class Consumer
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
+
+      class << self
+        # @return [Hash] current thread persistence scope hash with all the consumers
+        def all
+          # @note This does not need to be threadsafe (Hash) as it is always executed in a
+          # current thread context
+          Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+        end
+
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        def fetch(topic, partition)
+          # We always store a current instance for callback reasons
+          if topic.persistent
+            all[topic][partition] ||= topic.consumer.new
+          else
+            all[topic][partition] = topic.consumer.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/topic.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topic
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      # @param group_id [String] group id for which we fetch a topic representation
+      # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+      #   topic representation
+      # @return [Karafka::Routing::Topic] remapped topic representation that can be used further
+      #   on when working with given parameters
+      def self.fetch(group_id, raw_topic_name)
+        Thread.current[PERSISTENCE_SCOPE] ||= Hash.new { |hash, key| hash[key] = {} }
+
+        Thread.current[PERSISTENCE_SCOPE][group_id][raw_topic_name] ||= begin
+          # We map from incoming topic name, as it might be namespaced, etc.
+          # @see topic_mapper internal docs
+          mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+          Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
+  # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
     include Singleton
 
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
-      SIGINT SIGQUIT SIGTERM
+      SIGINT
+      SIGQUIT
+      SIGTERM
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -27,8 +29,7 @@ module Karafka
 
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
-      @callbacks = {}
-      HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
+      @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
@@ -56,7 +57,7 @@ module Karafka
     #   we have to spin up a new thread to do this
     def notice_signal(signal)
       Thread.new do
-        Karafka.monitor.notice(self.class, signal: signal)
+        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
       end
     end
   end

data/lib/karafka/responders/builder.rb

@@ -3,30 +3,31 @@
 module Karafka
   # Responders namespace encapsulates all the internal responder implementation parts
   module Responders
-    # Responders builder is used to find (based on the controller class name) a responder that
-    # match the controller. This is used when user does not provide a responder inside routing
-    # but he still names responder with the same convention (and namespaces) as controller
+    # Responders builder is used to finding (based on the consumer class name) a responder
+    # that match the consumer. We use it when user does not provide a responder inside routing,
+    # but he still names responder with the same convention (and namespaces) as consumer
+    #
     # @example Matching responder exists
-    #   Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
+    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
     # @example Matching responder does not exist
-    #   Karafka::Responder::Builder(NewBuildsController).build #=> nil
+    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
     class Builder
-      # @param controller_class [Karafka::BaseController, nil] descendant of
-      #   Karafka::BaseController
-      # @example Tries to find a responder that matches a given controller. If nothing found,
-      #   will return nil (nil is accepted, because it means that a given controller don't
+      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
+      #   Karafka::BaseConsumer
+      # @example Tries to find a responder that matches a given consumer. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given consumer don't
       #   pipe stuff further on)
-      def initialize(controller_class)
-        @controller_class = controller_class
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
       end
 
-      # Tries to figure out a responder based on a controller class name
+      # Tries to figure out a responder based on a consumer class name
       # @return [Class] Responder class (not an instance)
       # @return [nil] or nil if there's no matching responding class
       def build
         Helpers::ClassMatcher.new(
-          @controller_class,
-          from: 'Controller',
+          @consumer_class,
+          from: 'Consumer',
           to: 'Responder'
         ).match
       end

data/lib/karafka/responders/topic.rb

@@ -36,13 +36,20 @@ module Karafka
         @options[:registered] == true
       end
 
+      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
+      #   is safer and introduces less problems
+      def async?
+        @options.key?(:async) ? @options[:async] : false
+      end
+
       # @return [Hash] hash with this topic attributes and options
       def to_h
         {
           name: name,
           multiple_usage: multiple_usage?,
           required: required?,
-          registered: registered?
+          registered: registered?,
+          async: async?
         }
       end
     end