karafka 1.2.2 → 1.4.0.rc1

data/lib/karafka/connection/client.rb

@@ -7,7 +7,13 @@ module Karafka
     class Client
       extend Forwardable
 
-      def_delegator :kafka_consumer, :seek
+      %i[
+        seek
+        trigger_heartbeat
+        trigger_heartbeat!
+      ].each do |delegated_method|
+        def_delegator :kafka_consumer, delegated_method
+      end
 
       # Creates a queue consumer client that will pull the data from Kafka
       # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
@@ -20,40 +26,32 @@ module Karafka
       end
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
+      # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
+      #   the type of the fetcher that is being used
       # @note This will yield with raw messages - no preprocessing or reformatting.
       def fetch_loop
-        settings = ConfigAdapter.consuming(consumer_group)
+        settings = ApiAdapter.consumption(consumer_group)
 
         if consumer_group.batch_fetching
-          kafka_consumer.each_batch(*settings) { |batch| yield(batch.messages) }
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
         else
-          # always yield an array of messages, so we have consistent API (always a batch)
-          kafka_consumer.each_message(*settings) { |message| yield([message]) }
+          kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
         end
-      rescue Kafka::ProcessingError => error
+      # @note We catch only the processing errors as any other are considered critical (exceptions)
+      #   and should require a client restart with a backoff
+      rescue Kafka::ProcessingError => e
         # If there was an error during consumption, we have to log it, pause current partition
         # and process other things
         Karafka.monitor.instrument(
           'connection.client.fetch_loop.error',
           caller: self,
-          error: error.cause
-        )
-        pause(error.topic, error.partition)
-        retry
-        # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
-      rescue Exception => error
-        # rubocop:enable RescueException
-        Karafka.monitor.instrument(
-          'connection.client.fetch_loop.error',
-          caller: self,
-          error: error
+          error: e.cause
         )
+        pause(e.topic, e.partition)
         retry
       end
 
-      # Gracefuly stops topic consumption
+      # Gracefully stops topic consumption
       # @note Stopping running consumers without a really important reason is not recommended
       #   as until all the consumers are stopped, the server will keep running serving only
       #   part of the messages
@@ -66,18 +64,25 @@ module Karafka
       # @param topic [String] topic that we want to pause
       # @param partition [Integer] number partition that we want to pause
       def pause(topic, partition)
-        settings = ConfigAdapter.pausing(consumer_group)
-        timeout = settings[:timeout]
-        raise(Errors::InvalidPauseTimeout, timeout) unless timeout.positive?
-        kafka_consumer.pause(topic, partition, settings)
+        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
       end
 
-      # Marks a given message as consumed and commit the offsets
-      # @note In opposite to ruby-kafka, we commit the offset for each manual marking to be sure
-      #   that offset commit happen asap in case of a crash
+      # Marks given message as consumed
       # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
+      #   offsets time-interval based committing
       def mark_as_consumed(params)
-        kafka_consumer.mark_message_as_processed(params)
+        kafka_consumer.mark_message_as_processed(
+          *ApiAdapter.mark_message_as_processed(params)
+        )
+      end
+
+      # Marks a given message as consumed and commit the offsets in a blocking way
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method commits the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      def mark_as_consumed!(params)
+        mark_as_consumed(params)
         # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
         # before the automatic triggers have kicked in.
         kafka_consumer.commit_offsets
@@ -90,28 +95,23 @@ module Karafka
       # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
       #   that is set up to consume from topics of a given consumer group
       def kafka_consumer
-        @kafka_consumer ||= kafka.consumer(
-          *ConfigAdapter.consumer(consumer_group)
+        # @note We don't cache the connection internally because we cache kafka_consumer that uses
+        #   kafka client object instance
+        @kafka_consumer ||= Builder.call(consumer_group).consumer(
+          *ApiAdapter.consumer(consumer_group)
         ).tap do |consumer|
           consumer_group.topics.each do |topic|
-            consumer.subscribe(*ConfigAdapter.subscription(topic))
+            consumer.subscribe(*ApiAdapter.subscribe(topic))
           end
         end
       rescue Kafka::ConnectionError
-        # If we would not wait it would totally spam log file with failed
+        # If we would not wait it will spam log file with failed
         # attempts if Kafka is down
         sleep(consumer_group.reconnect_timeout)
-        # We don't log and just reraise - this will be logged
+        # We don't log and just re-raise - this will be logged
         # down the road
         raise
       end
-
-      # @return [Kafka] returns a Kafka
-      # @note We don't cache it internally because we cache kafka_consumer that uses kafka
-      #   object instance
-      def kafka
-        Kafka.new(*ConfigAdapter.client(consumer_group))
-      end
     end
   end
 end

data/lib/karafka/connection/listener.rb

@@ -16,9 +16,10 @@ module Karafka
 
       # Runs prefetch callbacks and executes the main listener fetch loop
       def call
-        Karafka::Callbacks.before_fetch_loop(
-          @consumer_group,
-          client
+        Karafka.monitor.instrument(
+          'connection.listener.before_fetch_loop',
+          consumer_group: @consumer_group,
+          client: client
         )
         fetch_loop
       end
@@ -26,28 +27,38 @@ module Karafka
       private
 
       # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [String] consumer group id
-      # @yieldparam [Array<Kafka::FetchedMessage>] kafka fetched messages
-      # @note This will yield with a raw message - no preprocessing or reformatting
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
       #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
-      #   won't crash the whole cluster. Here we mostly focus on catchin the exceptions related to
+      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far
       def fetch_loop
-        client.fetch_loop do |raw_messages|
-          # @note What happens here is a delegation of processing to a proper processor based
-          #   on the incoming messages characteristics
-          Karafka::Connection::Delegator.call(@consumer_group.id, raw_messages)
+        # @note What happens here is a delegation of processing to a proper processor based
+        #   on the incoming messages characteristics
+        client.fetch_loop do |raw_data, type|
+          Karafka.monitor.instrument('connection.listener.fetch_loop')
+
+          case type
+          when :message
+            MessageDelegator.call(@consumer_group.id, raw_data)
+          when :batch
+            BatchDelegator.call(@consumer_group.id, raw_data)
+          end
         end
         # This is on purpose - see the notes for this method
-        # rubocop:disable RescueException
+        # rubocop:disable Lint/RescueException
       rescue Exception => e
         Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
-        # rubocop:enable RescueException
-        @client&.stop
-        retry if @client
+        # rubocop:enable Lint/RescueException
+        # We can stop client without a problem, as it will reinitialize itself when running the
+        # `fetch_loop` again
+        @client.stop
+        # We need to clear the consumers cache for current connection when fatal error happens and
+        # we reset the connection. Otherwise for consumers with manual offset management, the
+        # persistence might have stored some data that would be reprocessed
+        Karafka::Persistence::Consumers.clear
+        sleep(@consumer_group.reconnect_timeout) && retry
       end
 
       # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic

data/lib/karafka/connection/message_delegator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of a single received message for which we listen to
+    # a proper processor
+    module MessageDelegator
+      class << self
+        # Delegates message (does something with it)
+        # It will either schedule or run a proper processor action for the incoming message
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_message [<Kafka::FetchedMessage>] raw message from kafka
+        # @note This should be looped to obtain a constant delegating of new messages
+        def call(group_id, kafka_message)
+          topic = Persistence::Topics.fetch(group_id, kafka_message.topic)
+          consumer = Persistence::Consumers.fetch(topic, kafka_message.partition)
+
+          Karafka.monitor.instrument(
+            'connection.message_delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_message: kafka_message
+          ) do
+            # @note We always get a single message within single delegator, which means that
+            # we don't care if user marked it as a batch consumed or not.
+            consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+              [kafka_message],
+              topic
+            )
+            consumer.call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/consumers/batch_metadata.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Consumers
+    # Brings the batch metadata into consumers that support batch_fetching
+    module BatchMetadata
+      attr_accessor :batch_metadata
+    end
+  end
+end

data/lib/karafka/consumers/callbacks.rb

@@ -16,28 +16,40 @@ module Karafka
         before_stop
       ].freeze
 
+      private_constant :TYPES
+
       # Class methods needed to make callbacks run
       module ClassMethods
         TYPES.each do |type|
-          # A Creates a callback wrapper
+          # Creates a callback wrapper
+          #
           # @param method_name [Symbol, String] method name or nil if we plan to provide a block
           # @yield A block with a code that should be executed before scheduling
-          define_method type do |method_name = nil, &block|
-            set_callback type, :before, method_name ? method_name : block
+          # @note We don't have to optimize the key fetching here as those are class methods that
+          #   are evaluated once upon start
+          define_method(type) do |method_name = nil, &block|
+            key = "consumers.#{Helpers::Inflector.map(to_s)}.#{type}"
+            Karafka::App.monitor.register_event(key)
+
+            Karafka::App.monitor.subscribe(key) do |event|
+              context = event[:context]
+
+              if method_name
+                context.send(method_name)
+              else
+                context.instance_eval(&block)
+              end
+            end
           end
         end
       end
 
-      # @param consumer_class [Class] consumer class that we extend with callbacks
-      def self.included(consumer_class)
-        consumer_class.class_eval do
-          extend ClassMethods
-          include ActiveSupport::Callbacks
-
-          # The call method is wrapped with a set of callbacks
-          # We won't run process if any of the callbacks throw abort
-          # @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
-          TYPES.each { |type| define_callbacks type }
+      class << self
+        # @param consumer_class [Class] consumer class that we extend with callbacks
+        def included(consumer_class)
+          consumer_class.class_eval do
+            extend ClassMethods
+          end
         end
       end
 
@@ -45,9 +57,14 @@ module Karafka
       # method of a proper backend. It is here because it interacts with the default Karafka
       # call flow and needs to be overwritten to support callbacks
       def call
-        run_callbacks :after_fetch do
-          process
+        if self.class.respond_to?(:after_fetch)
+          Karafka::App.monitor.instrument(
+            "consumers.#{Helpers::Inflector.map(self.class.to_s)}.after_fetch",
+            context: self
+          )
         end
+
+        process
       end
     end
   end

data/lib/karafka/consumers/includer.rb

@@ -3,47 +3,60 @@
 module Karafka
   # Additional functionalities for consumers
   module Consumers
-    # Module used to inject functionalities into a given consumer class, based on the consumer
+    # Module used to inject functionalities into a given consumer instance, based on the consumer
     # topic and its settings
     # We don't need all the behaviors in all the cases, so it is not worth having everything
     # in all the cases all the time
     module Includer
       class << self
-        # @param consumer_class [Class] consumer class, that will get some functionalities
-        #   based on the topic under which it operates
-        def call(consumer_class)
-          topic = consumer_class.topic
-
-          bind_backend(consumer_class, topic)
-          bind_params(consumer_class, topic)
-          bind_responders(consumer_class, topic)
+        # @param consumer [Karafka::BaseConsumer] consumer instance, that will get some
+        #   functionalities based on the topic under which it operates
+        def call(consumer)
+          topic = consumer.topic
+
+          bind_backend(consumer, topic)
+          bind_params(consumer, topic)
+          bind_batch_metadata(consumer, topic)
+          bind_responders(consumer, topic)
         end
 
         private
 
         # Figures out backend for a given consumer class, based on the topic backend and
         #   includes it into the consumer class
-        # @param consumer_class [Class] consumer class
+        # @param consumer [Karafka::BaseConsumer] consumer instance
         # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_backend(consumer_class, topic)
+        def bind_backend(consumer, topic)
           backend = Kernel.const_get("::Karafka::Backends::#{topic.backend.to_s.capitalize}")
-          consumer_class.include backend
+          consumer.extend(backend)
         end
 
         # Adds a single #params support for non batch processed topics
-        # @param consumer_class [Class] consumer class
+        # @param consumer [Karafka::BaseConsumer] consumer instance
         # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_params(consumer_class, topic)
+        def bind_params(consumer, topic)
           return if topic.batch_consuming
-          consumer_class.include SingleParams
+
+          consumer.extend(SingleParams)
+        end
+
+        # Adds an option to work with batch metadata for consumer instances that have
+        #   batch fetching enabled
+        # @param consumer [Karafka::BaseConsumer] consumer instance
+        # @param topic [Karafka::Routing::Topic] topic of a consumer class
+        def bind_batch_metadata(consumer, topic)
+          return unless topic.batch_fetching
+
+          consumer.extend(BatchMetadata)
         end
 
         # Adds responders support for topics and consumers with responders defined for them
-        # @param consumer_class [Class] consumer class
+        # @param consumer [Karafka::BaseConsumer] consumer instance
         # @param topic [Karafka::Routing::Topic] topic of a consumer class
-        def bind_responders(consumer_class, topic)
+        def bind_responders(consumer, topic)
           return unless topic.responder
-          consumer_class.include Responders
+
+          consumer.extend(Responders)
         end
       end
     end

data/lib/karafka/consumers/responders.rb

@@ -15,8 +15,8 @@ module Karafka
           data: data
         ) do
           # @note we build a new instance of responder each time, as a long-running (persisted)
-          #   consumers can respond multiple times during the lifecycle
-          topic.responder.new(topic.parser).call(*data)
+          #   consumers can respond multiple times during the life-cycle
+          topic.responder.new.call(*data)
         end
       end
     end

data/lib/karafka/contracts.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the validation contracts that we use to check input
+  module Contracts
+    # Regexp for validating format of groups and topics
+    # @note It is not nested inside of the contracts, as it is used by couple of them
+    TOPIC_REGEXP = /\A(\w|\-|\.)+\z/.freeze
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Contract with validation rules for Karafka configuration details
+    # @note There are many more configuration options inside of the
+    #   Karafka::Setup::Config model, but we don't validate them here as they are
+    #   validated per each route (topic + consumer_group) because they can be overwritten,
+    #   so we validate all of that once all the routes are defined and ready
+    class Config < Dry::Validation::Contract
+      params do
+        required(:client_id).filled(:str?, format?: Karafka::Contracts::TOPIC_REGEXP)
+        required(:shutdown_timeout) { (int? & gt?(0)) }
+        required(:consumer_mapper)
+        required(:topic_mapper)
+
+        optional(:backend).filled
+      end
+    end
+  end
+end