karafka 1.0.1 → 1.4.14

data/lib/karafka/params/params_batch.rb

@@ -3,38 +3,57 @@
 module Karafka
   module Params
     # Params batch represents a set of messages received from Kafka.
-    # @note Params internally are lazy loaded before first use. That way we can skip parsing
-    #   process if we have after_received that rejects some incoming messages without using params
-    #   It can be also used when handling really heavy data (in terms of parsing).
+    # @note Params internally are lazy loaded before first use. That way we can skip
+    #   deserialization process if we have after_fetch that rejects some incoming messages
+    #   without using params It can be also used when handling really heavy data.
     class ParamsBatch
       include Enumerable
 
-      # Builds up a params batch based on raw kafka messages
-      # @param messages_batch [Array<Kafka::FetchedMessage>] messages batch
-      # @param topic_parser [Class] topic parser for unparsing messages values
-      def initialize(messages_batch, topic_parser)
-        @params_batch = messages_batch.map do |message|
-          Karafka::Params::Params.build(message, topic_parser)
-        end
+      # @param params_array [Array<Karafka::Params::Params>] array with karafka params
+      # @return [Karafka::Params::ParamsBatch] lazy evaluated params batch object
+      def initialize(params_array)
+        @params_array = params_array
       end
 
-      # @yieldparam [Karafka::Params::Params] each parsed and loaded params instance
-      # @note Invocation of this method will cause loading and parsing each param after another.
-      #   If you want to get access without parsing, please access params_batch directly
+      # @yieldparam [Karafka::Params::Params] each params instance
+      # @note Invocation of this method will not cause loading and deserializing each param after
+      #   another.
       def each
-        @params_batch.each { |param| yield(param.retrieve!) }
+        @params_array.each { |param| yield(param) }
       end
 
       # @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
       #   can be used for batch insert, etc. Without invoking all, up until first use, they won't
-      #   be parsed
-      def parsed
-        each(&:itself)
+      #   be deserialized
+      def deserialize!
+        each(&:payload)
       end
 
-      # @return [Array<Karafka::Params::Params>] pure array with params (not parsed)
+      # @return [Array<Object>] array with deserialized payloads. This method can be useful when
+      #   we don't care about metadata and just want to extract all the data payloads from the
+      #   batch
+      def payloads
+        map(&:payload)
+      end
+
+      # @return [Karafka::Params::Params] first element
+      def first
+        @params_array.first
+      end
+
+      # @return [Karafka::Params::Params] last element
+      def last
+        @params_array.last
+      end
+
+      # @return [Integer] number of messages in the batch
+      def size
+        @params_array.size
+      end
+
+      # @return [Array<Karafka::Params::Params>] pure array with params
       def to_a
-        @params_batch
+        @params_array
       end
     end
   end

data/lib/karafka/patches/ruby_kafka.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for various other libs patches
+  module Patches
+    # Patches for Ruby Kafka gem
+    module RubyKafka
+      # This patch allows us to inject business logic in between fetches and before the consumer
+      # stop, so we can perform stop commit or anything else that we need since
+      # ruby-kafka fetch loop does not allow that directly
+      # We don't won't to use poll ruby-kafka api as it brings many more problems that we would
+      # have to take care of. That way, nothing like that ever happens but we get the control
+      # over the stopping process that we need (since we're the once that initiate it for each
+      # thread)
+      def consumer_loop
+        super do
+          consumers = Karafka::Persistence::Consumers
+                      .current
+                      .values
+                      .flat_map(&:values)
+                      .select { |consumer| consumer.class.respond_to?(:after_fetch) }
+
+          if Karafka::App.stopping?
+            publish_event(consumers, 'before_stop')
+            Karafka::Persistence::Client.read.stop
+          else
+            publish_event(consumers, 'before_poll')
+            yield
+            publish_event(consumers, 'after_poll')
+          end
+        end
+      end
+
+      private
+
+      # Notifies consumers about particular events happening
+      # @param consumers [Array<Object>] all consumers that want to be notified about an event
+      # @param event_name [String] name of the event that happened
+      def publish_event(consumers, event_name)
+        consumers.each do |consumer|
+          key = "consumers.#{Helpers::Inflector.map(consumer.class.to_s)}.#{event_name}"
+          Karafka::App.monitor.instrument(key, context: consumer)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/client.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Persistence layer to store current thread messages consumer client for further use
+    class Client
+      # Thread.current key under which we store current thread messages consumer client
+      PERSISTENCE_SCOPE = :client
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @param client [Karafka::Connection::Client] messages consumer client of
+        #   a current thread
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        def write(client)
+          Thread.current[PERSISTENCE_SCOPE] = client
+        end
+
+        # @return [Karafka::Connection::Client] persisted messages consumer client
+        # @raise [Karafka::Errors::MissingClientError] raised when no thread messages consumer
+        #   client but we try to use it anyway
+        def read
+          Thread.current[PERSISTENCE_SCOPE] || raise(Errors::MissingClientError)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/consumers.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module used to provide a persistent cache layer for Karafka components that need to be
+  # shared inside of a same thread
+  module Persistence
+    # Module used to provide a persistent cache across batch requests for a given
+    # topic and partition to store some additional details when the persistent mode
+    # for a given topic is turned on
+    class Consumers
+      # Thread.current scope under which we store consumers data
+      PERSISTENCE_SCOPE = :consumers
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @return [Hash] current thread's persistence scope hash with all the consumers
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
+        end
+
+        # Used to build (if block given) and/or fetch a current consumer instance that will be
+        #   used to process messages from a given topic and partition
+        # @param topic [Karafka::Routing::Topic] topic instance for which we might cache
+        # @param partition [Integer] number of partition for which we want to cache
+        # @return [Karafka::BaseConsumer] base consumer descendant
+        def fetch(topic, partition)
+          current[topic][partition] ||= topic.consumer.new(topic)
+        end
+
+        # Removes all persisted instances of consumers from the consumer cache
+        # @note This is used to reload consumers instances when code reloading in development mode
+        #   is present. This should not be used in production.
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/persistence/topics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Persistence
+    # Local cache for routing topics
+    # We use it in order not to build string instances and remap incoming topic upon each
+    # message / message batches received
+    class Topics
+      # Thread.current scope under which we store topics data
+      PERSISTENCE_SCOPE = :topics
+
+      private_constant :PERSISTENCE_SCOPE
+
+      class << self
+        # @return [Concurrent::Hash] hash with all the topics from given groups
+        def current
+          Thread.current[PERSISTENCE_SCOPE] ||= Concurrent::Hash.new do |hash, key|
+            hash[key] = Concurrent::Hash.new
+          end
+        end
+
+        # @param group_id [String] group id for which we fetch a topic representation
+        # @param raw_topic_name [String] raw topic name (before remapping) for which we fetch a
+        #   topic representation
+        # @return [Karafka::Routing::Topics] remapped topic representation that can be used further
+        #   on when working with given parameters
+        def fetch(group_id, raw_topic_name)
+          current[group_id][raw_topic_name] ||= begin
+            # We map from incoming topic name, as it might be namespaced, etc.
+            # @see topic_mapper internal docs
+            mapped_topic_name = Karafka::App.config.topic_mapper.incoming(raw_topic_name)
+            Routing::Router.find("#{group_id}_#{mapped_topic_name}")
+          end
+        end
+
+        # Clears the whole topics cache for all the threads
+        # This is used for in-development code reloading as we need to get rid of all the
+        # preloaded and cached instances of objects to make it work
+        def clear
+          Thread
+            .list
+            .select { |thread| thread[PERSISTENCE_SCOPE] }
+            .each { |thread| thread[PERSISTENCE_SCOPE].clear }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/process.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 module Karafka
-  # Class used to catch signals from ruby Signal class in order to manage Karafka shutdown
+  # Class used to catch signals from ruby Signal class in order to manage Karafka stop
   # @note There might be only one process - this class is a singleton
   class Process
-    include Singleton
-
     # Signal types that we handle
     HANDLED_SIGNALS = %i[
-      SIGINT SIGQUIT SIGTERM
+      SIGINT
+      SIGQUIT
+      SIGTERM
     ].freeze
 
     HANDLED_SIGNALS.each do |signal|
@@ -27,16 +27,13 @@ module Karafka
 
     # Creates an instance of process and creates empty hash for callbacks
     def initialize
-      @callbacks = {}
-      HANDLED_SIGNALS.each { |signal| @callbacks[signal] = [] }
+      @callbacks = Hash.new { |hsh, key| hsh[key] = [] }
     end
 
     # Method catches all HANDLED_SIGNALS and performs appropriate callbacks (if defined)
     # @note If there are no callbacks, this method will just ignore a given signal that was sent
-    # @yield [Block] block of code that we want to execute and supervise
     def supervise
       HANDLED_SIGNALS.each { |signal| trap_signal(signal) }
-      yield
     end
 
     private
@@ -56,7 +53,7 @@ module Karafka
     #   we have to spin up a new thread to do this
     def notice_signal(signal)
       Thread.new do
-        Karafka.monitor.notice(self.class, signal: signal)
+        Karafka.monitor.instrument('process.notice_signal', caller: self, signal: signal)
       end
     end
   end

data/lib/karafka/responders/builder.rb

@@ -3,30 +3,31 @@
 module Karafka
   # Responders namespace encapsulates all the internal responder implementation parts
   module Responders
-    # Responders builder is used to find (based on the controller class name) a responder that
-    # match the controller. This is used when user does not provide a responder inside routing
-    # but he still names responder with the same convention (and namespaces) as controller
+    # Responders builder is used for finding (based on the consumer class name) a responder
+    # that match the consumer. We use it when user does not provide a responder inside routing,
+    # but he still names responder with the same convention (and namespaces) as consumer
+    #
     # @example Matching responder exists
-    #   Karafka::Responder::Builder(NewEventsController).build #=> NewEventsResponder
+    #   Karafka::Responder::Builder(NewEventsConsumer).build #=> NewEventsResponder
     # @example Matching responder does not exist
-    #   Karafka::Responder::Builder(NewBuildsController).build #=> nil
+    #   Karafka::Responder::Builder(NewBuildsConsumer).build #=> nil
     class Builder
-      # @param controller_class [Karafka::BaseController, nil] descendant of
-      #   Karafka::BaseController
-      # @example Tries to find a responder that matches a given controller. If nothing found,
-      #   will return nil (nil is accepted, because it means that a given controller don't
+      # @param consumer_class [Karafka::BaseConsumer, nil] descendant of
+      #   Karafka::BaseConsumer
+      # @example Tries to find a responder that matches a given consumer. If nothing found,
+      #   will return nil (nil is accepted, because it means that a given consumer don't
       #   pipe stuff further on)
-      def initialize(controller_class)
-        @controller_class = controller_class
+      def initialize(consumer_class)
+        @consumer_class = consumer_class
       end
 
-      # Tries to figure out a responder based on a controller class name
+      # Tries to figure out a responder based on a consumer class name
       # @return [Class] Responder class (not an instance)
       # @return [nil] or nil if there's no matching responding class
       def build
         Helpers::ClassMatcher.new(
-          @controller_class,
-          from: 'Controller',
+          @consumer_class,
+          from: 'Consumer',
           to: 'Responder'
         ).match
       end

data/lib/karafka/responders/topic.rb

@@ -7,8 +7,6 @@ module Karafka
     #   Karafka::Responders::Topic.new(:topic_name, {}) #=> #<Karafka::Responders::Topic...
     # @example Define optional topic
     #   Karafka::Responders::Topic.new(:topic_name, required: false)
-    # @example Define topic that on which we want to respond multiple times
-    #   Karafka::Responders::Topic.new(:topic_name, multiple_usage: true)
     class Topic
       # Name of the topic on which we want to respond
       attr_reader :name
@@ -26,23 +24,30 @@ module Karafka
         @options.key?(:required) ? @options[:required] : true
       end
 
-      # @return [Boolean] do we expect to use it multiple times in a single respond flow
-      def multiple_usage?
-        @options[:multiple_usage] || false
-      end
-
       # @return [Boolean] was usage of this topic registered or not
       def registered?
         @options[:registered] == true
       end
 
+      # @return [Class] Class to use to serialize messages for this topic
+      def serializer
+        @options[:serializer]
+      end
+
+      # @return [Boolean] do we want to use async producer. Defaults to false as the sync producer
+      #   is safer and introduces less problems
+      def async?
+        @options.key?(:async) ? @options[:async] : false
+      end
+
       # @return [Hash] hash with this topic attributes and options
       def to_h
         {
           name: name,
-          multiple_usage: multiple_usage?,
           required: required?,
-          registered: registered?
+          registered: registered?,
+          serializer: serializer,
+          async: async?
         }
       end
     end

data/lib/karafka/routing/builder.rb

@@ -6,29 +6,43 @@ module Karafka
     # @example Build a simple (most common) route
     #   consumers do
     #     topic :new_videos do
-    #       controller NewVideosController
+    #       consumer NewVideosConsumer
     #     end
     #   end
-    class Builder < Array
-      include Singleton
+    class Builder < Concurrent::Array
+      # Consumer group consistency checking contract
+      CONTRACT = Karafka::Contracts::ConsumerGroup.new.freeze
+
+      private_constant :CONTRACT
+
+      def initialize
+        super
+        @draws = Concurrent::Array.new
+      end
 
       # Used to draw routes for Karafka
+      # @param block [Proc] block we will evaluate within the builder context
+      # @yield Evaluates provided block in a builder context so we can describe routes
+      # @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
+      #   doesn't match with the config contract
       # @note After it is done drawing it will store and validate all the routes to make sure that
       #   they are correct and that there are no topic/group duplications (this is forbidden)
-      # @yield Evaluates provided block in a builder context so we can describe routes
       # @example
       #   draw do
       #     topic :xyz do
       #     end
       #   end
       def draw(&block)
+        @draws << block
+
         instance_eval(&block)
 
         each do |consumer_group|
           hashed_group = consumer_group.to_h
-          validation_result = Karafka::Schemas::ConsumerGroup.call(hashed_group)
-          return if validation_result.success?
-          raise Errors::InvalidConfiguration, validation_result.errors
+          validation_result = CONTRACT.call(hashed_group)
+          next if validation_result.success?
+
+          raise Errors::InvalidConfigurationError, validation_result.errors.to_h
         end
       end
 
@@ -39,18 +53,33 @@ module Karafka
         select(&:active?)
       end
 
+      # Clears the builder and the draws memory
+      def clear
+        @draws.clear
+        super
+      end
+
+      # Redraws all the routes for the in-process code reloading.
+      # @note This won't allow registration of new topics without process restart but will trigger
+      #   cache invalidation so all the classes, etc are re-fetched after code reload
+      def reload
+        draws = @draws.dup
+        clear
+        draws.each { |block| draw(&block) }
+      end
+
       private
 
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
-      # @yield Evaluates a given block in a consumer group context
+      # @param block [Proc] proc that should be executed in the proxy context
       def consumer_group(group_id, &block)
         consumer_group = ConsumerGroup.new(group_id.to_s)
         self << Proxy.new(consumer_group, &block).target
       end
 
       # @param topic_name [String, Symbol] name of a topic from which we want to consumer
-      # @yield Evaluates a given block in a topic context
+      # @param block [Proc] proc we want to evaluate in the topic context
       def topic(topic_name, &block)
         consumer_group(topic_name) do
           topic(topic_name, &block).tap(&:build)

data/lib/karafka/routing/consumer_group.rb

@@ -8,9 +8,11 @@ module Karafka
     class ConsumerGroup
       extend Helpers::ConfigRetriever
 
-      attr_reader :topics
-      attr_reader :id
-      attr_reader :name
+      attr_reader(
+        :topics,
+        :id,
+        :name
+      )
 
       # @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
       #   yet have an application client_id namespace, this will be added here by default.
@@ -29,7 +31,7 @@ module Karafka
 
       # Builds a topic representation inside of a current consumer group route
       # @param name [String, Symbol] name of topic to which we want to subscribe
-      # @yield Evaluates a given block in a topic context
+      # @param block [Proc] block that we want to evaluate in the topic context
       # @return [Karafka::Routing::Topic] newly built topic instance
       def topic=(name, &block)
         topic = Topic.new(name, self)

data/lib/karafka/routing/consumer_mapper.rb

@@ -4,29 +4,30 @@ module Karafka
   module Routing
     # Default consumer mapper that builds consumer ids based on app id and consumer group name
     # Different mapper can be used in case of preexisting consumer names or for applying
-    # other naming conventions not compatible wiih Karafkas client_id + consumer name concept
+    # other naming conventions not compatible with Karafka client_id + consumer name concept
     #
     # @example Mapper for using consumer groups without a client_id prefix
-    #   module MyMapper
-    #     def self.call(raw_consumer_group_name)
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
     #       raw_consumer_group_name
     #     end
     #   end
     #
     # @example Mapper for replacing "_" with "." in topic names
-    #   module MyMapper
-    #     def self.call(raw_consumer_group_name)
+    #   class MyMapper
+    #     def call(raw_consumer_group_name)
     #       [
-    #         Karafka::App.config.client_id.to_s.underscope,
+    #         Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s),
     #         raw_consumer_group_name
     #       ].join('_').gsub('_', '.')
     #     end
     #   end
-    module ConsumerMapper
+    class ConsumerMapper
       # @param raw_consumer_group_name [String, Symbol] string or symbolized consumer group name
       # @return [String] remapped final consumer group name
-      def self.call(raw_consumer_group_name)
-        "#{Karafka::App.config.client_id.to_s.underscore}_#{raw_consumer_group_name}"
+      def call(raw_consumer_group_name)
+        client_name = Karafka::Helpers::Inflector.map(Karafka::App.config.client_id.to_s)
+        "#{client_name}_#{raw_consumer_group_name}"
       end
     end
   end

data/lib/karafka/routing/proxy.rb

@@ -14,22 +14,31 @@ module Karafka
         !
       ].freeze
 
+      private_constant :IGNORED_POSTFIXES
+
       # @param target [Object] target object to which we proxy any DSL call
-      # @yield Evaluates block in the proxy context
+      # @param block [Proc] block that we want to evaluate in the proxy context
       def initialize(target, &block)
         @target = target
         instance_eval(&block)
       end
 
       # Translates the no "=" DSL of routing into elements assignments on target
+      # @param method_name [Symbol] name of the missing method
+      # @param arguments [Array] array with it's arguments
+      # @param block [Proc] block provided to the method
       def method_missing(method_name, *arguments, &block)
         return super unless respond_to_missing?(method_name)
+
         @target.public_send(:"#{method_name}=", *arguments, &block)
       end
 
       # Tells whether or not a given element exists on the target
+      # @param method_name [Symbol] name of the missing method
+      # @param include_private [Boolean] should we include private in the check as well
       def respond_to_missing?(method_name, include_private = false)
         return false if IGNORED_POSTFIXES.any? { |postfix| method_name.to_s.end_with?(postfix) }
+
         @target.respond_to?(:"#{method_name}=", include_private) || super
       end
     end

data/lib/karafka/routing/router.rb

@@ -3,7 +3,7 @@
 module Karafka
   # Namespace for all elements related to requests routing
   module Routing
-    # Karafka framework Router for routing incoming messages to proper controllers
+    # Karafka framework Router for routing incoming messages to proper consumers
     # @note Since Kafka does not provide namespaces or modules for topics, they all have "flat"
     #  structure so all the routes are being stored in a single level array
     module Router

data/lib/karafka/routing/topic.rb

@@ -7,9 +7,12 @@ module Karafka
     # It is a part of Karafka's DSL
     class Topic
       extend Helpers::ConfigRetriever
+      extend Forwardable
 
       attr_reader :id, :consumer_group
-      attr_accessor :controller
+      attr_accessor :consumer
+
+      def_delegator :@consumer_group, :batch_fetching
 
       # @param [String, Symbol] name of a topic on which we want to listen
       # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
@@ -19,7 +22,7 @@ module Karafka
         @attributes = {}
         # @note We use identifier related to the consumer group that owns a topic, because from
         #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across mutliple Kafkas
+        #   have same topic name across multiple Kafkas
         @id = "#{consumer_group.id}_#{@name}"
       end
 
@@ -29,20 +32,13 @@ module Karafka
       # example for Sidekiq
       def build
         Karafka::AttributesMap.topic.each { |attr| send(attr) }
-        controller&.topic = self
         self
       end
 
       # @return [Class, nil] Class (not an instance) of a responder that should respond from
-      #   controller back to Kafka (usefull for piping dataflows)
+      #   consumer back to Kafka (useful for piping data flows)
       def responder
-        @responder ||= Karafka::Responders::Builder.new(controller).build
-      end
-
-      # @return [Class] Parser class (not instance) that we want to use to unparse Kafka messages
-      # @note If not provided - will use Json as default
-      def parser
-        @parser ||= Karafka::Parsers::Json
+        @responder ||= Karafka::Responders::Builder.new(consumer).build
       end
 
       Karafka::AttributesMap.topic.each do |attribute|
@@ -58,7 +54,7 @@ module Karafka
 
         Hash[map].merge!(
           id: id,
-          controller: controller
+          consumer: consumer
         )
       end
     end