karafka 1.3.0

data/lib/karafka/connection/api_adapter.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace for all the things related to Kafka connection
+  module Connection
+    # Mapper used to convert our internal settings into ruby-kafka settings based on their
+    # API requirements.
+    # Since ruby-kafka has more and more options and there are few "levels" on which
+    # we have to apply them (despite the fact, that in Karafka you configure all of it
+    # in one place), we have to remap it into what ruby-kafka driver requires
+    # @note The good thing about Kafka.new method is that it ignores all options that
+    #   do nothing. So we don't have to worry about injecting our internal settings
+    #   into the client and breaking stuff
+    module ApiAdapter
+      class << self
+        # Builds all the configuration settings for Kafka.new method
+        # @return [Array<Hash>] Array with all the client arguments including hash with all
+        #   the settings required by Kafka.new method
+        # @note We return array, so we can inject any arguments we want, in case of changes in the
+        #   raw driver
+        def client
+          # This one is a default that takes all the settings except special
+          # cases defined in the map
+          settings = {
+            logger: ::Karafka.logger,
+            client_id: ::Karafka::App.config.client_id
+          }
+
+          kafka_configs.each do |setting_name, setting_value|
+            # All options for config adapter should be ignored as we're just interested
+            # in what is left, as we want to pass all the options that are "typical"
+            # and not listed in the api_adapter special cases mapping. All the values
+            # from the api_adapter mapping go somewhere else, not to the client directly
+            next if AttributesMap.api_adapter.values.flatten.include?(setting_name)
+
+            settings[setting_name] = setting_value
+          end
+
+          settings_hash = sanitize(settings)
+
+          # Normalization for the way Kafka::Client accepts arguments from  0.5.3
+          [settings_hash.delete(:seed_brokers), settings_hash]
+        end
+
+        # Builds all the configuration settings for kafka#consumer method
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] array with all the consumer arguments including hash with all
+        #   the settings required by Kafka#consumer
+        def consumer(consumer_group)
+          settings = { group_id: consumer_group.id }
+          settings = fetch_for(:consumer, consumer_group, settings)
+          [sanitize(settings)]
+        end
+
+        # Builds all the configuration settings for kafka consumer consume_each_batch and
+        #   consume_each_message methods
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array<Hash>] Array with all the arguments required by consuming method
+        #   including hash with all the settings required by
+        #   Kafka::Consumer#consume_each_message and Kafka::Consumer#consume_each_batch method
+        def consumption(consumer_group)
+          [
+            sanitize(
+              fetch_for(
+                :consumption,
+                consumer_group,
+                automatically_mark_as_processed: consumer_group.automatically_mark_as_consumed
+              )
+            )
+          ]
+        end
+
+        # Builds all the configuration settings for kafka consumer#subscribe method
+        # @param topic [Karafka::Routing::Topic] topic that holds details for a given subscription
+        # @return [Hash] hash with all the settings required by kafka consumer#subscribe method
+        def subscribe(topic)
+          settings = fetch_for(:subscribe, topic)
+          [Karafka::App.config.topic_mapper.outgoing(topic.name), sanitize(settings)]
+        end
+
+        # Builds all the configuration settings required by kafka consumer#pause method
+        # @param topic [String] topic that we want to pause
+        # @param partition [Integer] number partition that we want to pause
+        # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group details
+        # @return [Array] array with all the details required to pause kafka consumer
+        def pause(topic, partition, consumer_group)
+          [
+            Karafka::App.config.topic_mapper.outgoing(topic),
+            partition,
+            {
+              timeout: consumer_group.pause_timeout,
+              max_timeout: consumer_group.pause_max_timeout,
+              exponential_backoff: consumer_group.pause_exponential_backoff
+            }
+          ]
+        end
+
+        # Remaps topic details taking the topic mapper feature into consideration.
+        # @param params [Karafka::Params::Params] params instance
+        # @return [Array] array with all the details needed by ruby-kafka to mark message
+        #   as processed
+        # @note When default empty topic mapper is used, no need for any conversion as the
+        #   internal and external format are exactly the same
+        def mark_message_as_processed(params)
+          # Majority of users don't use custom topic mappers. No need to change anything when it
+          # is a default mapper that does not change anything. Only some cloud providers require
+          # topics to be remapped
+          return [params] if Karafka::App.config.topic_mapper.is_a?(Karafka::Routing::TopicMapper)
+
+          # @note We don't use tap as it is around 13% slower than non-dup version
+          dupped = params.dup
+          dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
+          [dupped]
+        end
+
+        private
+
+        # Fetches proper settings for a given map namespace
+        # @param namespace_key [Symbol] namespace from attributes map config adapter hash
+        # @param route_layer [Object] route topic or consumer group
+        # @param preexisting_settings [Hash] hash with some preexisting settings that might have
+        #   been loaded in a different way
+        def fetch_for(namespace_key, route_layer, preexisting_settings = {})
+          kafka_configs.each_key do |setting_name|
+            # Ignore settings that are not related to our namespace
+            next unless AttributesMap.api_adapter[namespace_key].include?(setting_name)
+
+            # Ignore settings that are already initialized
+            # In case they are in preexisting settings fetched differently
+            next if preexisting_settings.key?(setting_name)
+
+            # Fetch all the settings from a given layer object. Objects can handle the fallback
+            # to the kafka settings, so
+            preexisting_settings[setting_name] = route_layer.send(setting_name)
+          end
+
+          preexisting_settings
+        end
+
+        # Removes nil containing keys from the final settings so it can use Kafkas driver
+        #   defaults for those
+        # @param settings [Hash] settings that may contain nil values
+        # @return [Hash] settings without nil using keys (non of karafka options should be nil)
+        def sanitize(settings)
+          settings.reject { |_key, value| value.nil? }
+        end
+
+        # @return [Hash] Kafka config details as a hash
+        def kafka_configs
+          ::Karafka::App.config.kafka.to_h
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/batch_delegator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of batch received messages for which we listen to
+    # a proper processor
+    module BatchDelegator
+      class << self
+        # Delegates messages (does something with them)
+        # It will either schedule or run a proper processor action for messages
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
+        # @note This should be looped to obtain a constant delegating of new messages
+        def call(group_id, kafka_batch)
+          topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
+          consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
+
+          Karafka.monitor.instrument(
+            'connection.batch_delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_batch: kafka_batch
+          ) do
+            # Due to how ruby-kafka is built, we have the metadata that is stored on the batch
+            # level only available for batch consuming
+            consumer.metadata = Params::Builders::Metadata.from_kafka_batch(kafka_batch, topic)
+            kafka_messages = kafka_batch.messages
+
+            # Depending on a case (persisted or not) we might use new consumer instance per
+            # each batch, or use the same one for all of them (for implementing buffering, etc.)
+            if topic.batch_consuming
+              consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                kafka_messages,
+                topic
+              )
+              consumer.call
+            else
+              kafka_messages.each do |kafka_message|
+                consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+                  [kafka_message],
+                  topic
+                )
+                consumer.call
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/builder.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Builder used to construct Kafka client
+    module Builder
+      class << self
+        # Builds a Kafka::Client instance that we use to work with Kafka cluster
+        # @return [::Kafka::Client] returns a Kafka client
+        def call
+          Kafka.new(*ApiAdapter.client)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class used as a wrapper around Ruby-Kafka client to simplify additional
+    # features that we provide/might provide in future and to hide the internal implementation
+    class Client
+      extend Forwardable
+
+      %i[
+        seek
+        trigger_heartbeat
+        trigger_heartbeat!
+      ].each do |delegated_method|
+        def_delegator :kafka_consumer, delegated_method
+      end
+
+      # Creates a queue consumer client that will pull the data from Kafka
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
+      #   we create a client
+      # @return [Karafka::Connection::Client] group consumer that can subscribe to
+      #   multiple topics
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+        Persistence::Client.write(self)
+      end
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
+      #   the type of the fetcher that is being used
+      # @note This will yield with raw messages - no preprocessing or reformatting.
+      def fetch_loop
+        settings = ApiAdapter.consumption(consumer_group)
+
+        if consumer_group.batch_fetching
+          kafka_consumer.each_batch(*settings) { |batch| yield(batch, :batch) }
+        else
+          kafka_consumer.each_message(*settings) { |message| yield(message, :message) }
+        end
+      # @note We catch only the processing errors as any other are considered critical (exceptions)
+      #   and should require a client restart with a backoff
+      rescue Kafka::ProcessingError => e
+        # If there was an error during consumption, we have to log it, pause current partition
+        # and process other things
+        Karafka.monitor.instrument(
+          'connection.client.fetch_loop.error',
+          caller: self,
+          error: e.cause
+        )
+        pause(e.topic, e.partition)
+        retry
+      end
+
+      # Gracefully stops topic consumption
+      # @note Stopping running consumers without a really important reason is not recommended
+      #   as until all the consumers are stopped, the server will keep running serving only
+      #   part of the messages
+      def stop
+        @kafka_consumer&.stop
+        @kafka_consumer = nil
+      end
+
+      # Pauses fetching and consumption of a given topic partition
+      # @param topic [String] topic that we want to pause
+      # @param partition [Integer] number partition that we want to pause
+      def pause(topic, partition)
+        kafka_consumer.pause(*ApiAdapter.pause(topic, partition, consumer_group))
+      end
+
+      # Marks given message as consumed
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
+      #   offsets time-interval based committing
+      def mark_as_consumed(params)
+        kafka_consumer.mark_message_as_processed(
+          *ApiAdapter.mark_message_as_processed(params)
+        )
+      end
+
+      # Marks a given message as consumed and commit the offsets in a blocking way
+      # @param [Karafka::Params::Params] params message that we want to mark as processed
+      # @note This method commits the offset for each manual marking to be sure
+      #   that offset commit happen asap in case of a crash
+      def mark_as_consumed!(params)
+        mark_as_consumed(params)
+        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
+        # before the automatic triggers have kicked in.
+        kafka_consumer.commit_offsets
+      end
+
+      private
+
+      attr_reader :consumer_group
+
+      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
+      #   that is set up to consume from topics of a given consumer group
+      def kafka_consumer
+        # @note We don't cache the connection internally because we cache kafka_consumer that uses
+        #   kafka client object instance
+        @kafka_consumer ||= Builder.call.consumer(
+          *ApiAdapter.consumer(consumer_group)
+        ).tap do |consumer|
+          consumer_group.topics.each do |topic|
+            consumer.subscribe(*ApiAdapter.subscribe(topic))
+          end
+        end
+      rescue Kafka::ConnectionError
+        # If we would not wait it will spam log file with failed
+        # attempts if Kafka is down
+        sleep(consumer_group.reconnect_timeout)
+        # We don't log and just re-raise - this will be logged
+        # down the road
+        raise
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # A single listener that listens to incoming messages from a single route
+    # @note It does not loop on itself - it needs to be executed in a loop
+    # @note Listener itself does nothing with the message - it will return to the block
+    #   a raw Kafka::FetchedMessage
+    class Listener
+      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
+      #   on what topics and with what settings should we listen
+      # @return [Karafka::Connection::Listener] listener instance
+      def initialize(consumer_group)
+        @consumer_group = consumer_group
+      end
+
+      # Runs prefetch callbacks and executes the main listener fetch loop
+      def call
+        Karafka.monitor.instrument(
+          'connection.listener.before_fetch_loop',
+          consumer_group: @consumer_group,
+          client: client
+        )
+        fetch_loop
+      end
+
+      private
+
+      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # @note We catch all the errors here, so they don't affect other listeners (or this one)
+      #   so we will be able to listen and consume other incoming messages.
+      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
+      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
+      #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
+      #   propagate this far
+      def fetch_loop
+        # @note What happens here is a delegation of processing to a proper processor based
+        #   on the incoming messages characteristics
+        client.fetch_loop do |raw_data, type|
+          Karafka.monitor.instrument('connection.listener.fetch_loop')
+
+          case type
+          when :message
+            MessageDelegator.call(@consumer_group.id, raw_data)
+          when :batch
+            BatchDelegator.call(@consumer_group.id, raw_data)
+          end
+        end
+        # This is on purpose - see the notes for this method
+        # rubocop:disable RescueException
+      rescue Exception => e
+        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
+        # rubocop:enable RescueException
+        # We can stop client without a problem, as it will reinitialize itself when running the
+        # `fetch_loop` again
+        @client.stop
+        # We need to clear the consumers cache for current connection when fatal error happens and
+        # we reset the connection. Otherwise for consumers with manual offset management, the
+        # persistence might have stored some data that would be reprocessed
+        Karafka::Persistence::Consumers.clear
+        sleep(@consumer_group.reconnect_timeout) && retry
+      end
+
+      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
+      #   consumption
+      def client
+        @client ||= Client.new(@consumer_group)
+      end
+    end
+  end
+end

data/lib/karafka/connection/message_delegator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Class that delegates processing of a single received message for which we listen to
+    # a proper processor
+    module MessageDelegator
+      class << self
+        # Delegates message (does something with it)
+        # It will either schedule or run a proper processor action for the incoming message
+        # @param group_id [String] group_id of a group from which a given message came
+        # @param kafka_message [<Kafka::FetchedMessage>] raw message from kafka
+        # @note This should be looped to obtain a constant delegating of new messages
+        def call(group_id, kafka_message)
+          topic = Persistence::Topics.fetch(group_id, kafka_message.topic)
+          consumer = Persistence::Consumers.fetch(topic, kafka_message.partition)
+
+          Karafka.monitor.instrument(
+            'connection.message_delegator.call',
+            caller: self,
+            consumer: consumer,
+            kafka_message: kafka_message
+          ) do
+            # @note We always get a single message within single delegator, which means that
+            # we don't care if user marked it as a batch consumed or not.
+            consumer.params_batch = Params::Builders::ParamsBatch.from_kafka_messages(
+              [kafka_message],
+              topic
+            )
+            consumer.call
+          end
+        end
+      end
+    end
+  end
+end