karafka 1.4.9 → 2.0.0.alpha1

data/lib/karafka/connection/client.rb

@@ -1,119 +1,305 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for Kafka connection related logic
   module Connection
-    # Class used as a wrapper around Ruby-Kafka client to simplify additional
-    # features that we provide/might provide in future and to hide the internal implementation
+    # An abstraction layer on top of the rdkafka consumer.
+    #
+    # It is threadsafe and provides some security measures so we won't end up operating on a
+    # closed consumer instance as it causes Ruby VM process to crash.
     class Client
-      extend Forwardable
-
-      %i[
-        seek
-        trigger_heartbeat
-        trigger_heartbeat!
-      ].each do |delegated_method|
-        def_delegator :kafka_consumer, delegated_method
-      end
-
-      # Creates a queue consumer client that will pull the data from Kafka
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group for which
-      #   we create a client
-      # @return [Karafka::Connection::Client] group consumer that can subscribe to
-      #   multiple topics
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
-        Persistence::Client.write(self)
-      end
-
-      # Opens connection, gets messages and calls a block for each of the incoming messages
-      # @yieldparam [Array<Kafka::FetchedMessage>, Symbol] kafka response with an info about
-      #   the type of the fetcher that is being used
-      # @note This will yield with raw messages - no preprocessing or reformatting.
-      def fetch_loop
-        settings = ApiAdapter.consumption(consumer_group)
-
-        if consumer_group.batch_fetching
-          kafka_consumer.each_batch(**settings) { |batch| yield(batch, :batch) }
-        else
-          kafka_consumer.each_message(**settings) { |message| yield(message, :message) }
+      attr_reader :rebalance_manager
+
+      # @return [String] underlying consumer name
+      # @note Consumer name may change in case we regenerate it
+      attr_reader :name
+
+      # How many times should we retry polling in case of a failure
+      MAX_POLL_RETRIES = 10
+
+      private_constant :MAX_POLL_RETRIES
+
+      # Creates a new consumer instance.
+      #
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
+      #   with all the configuration details needed for us to create a client
+      # @return [Karafka::Connection::Rdk::Consumer]
+      def initialize(subscription_group)
+        # Name is set when we build consumer
+        @name = ''
+        @mutex = Mutex.new
+        @closed = false
+        @subscription_group = subscription_group
+        @buffer = MessagesBuffer.new
+        @rebalance_manager = RebalanceManager.new
+        @kafka = build_consumer
+        # Marks if we need to offset. If we did not store offsets, we should not commit the offset
+        # position as it will crash rdkafka
+        @offsetting = false
+      end
+
+      # Fetches messages within boundaries defined by the settings (time, size, topics, etc).
+      #
+      # @return [Karafka::Connection::MessagesBuffer] messages buffer that holds messages per topic
+      #   partition
+      # @note This method should not be executed from many threads at the same time
+      def batch_poll
+        time_poll = TimeTrackers::Poll.new(@subscription_group.max_wait_time)
+        time_poll.start
+
+        @buffer.clear
+
+        loop do
+          # Don't fetch more messages if we do not have any time left
+          break if time_poll.exceeded?
+          # Don't fetch more messages if we've fetched max as we've wanted
+          break if @buffer.size >= @subscription_group.max_messages
+
+          # Fetch message within our time boundaries
+          message = poll(time_poll.remaining)
+
+          # If there are no more messages, return what we have
+          break unless message
+
+          @buffer << message
+
+          # Track time spent on all of the processing and polling
+          time_poll.checkpoint
         end
-      # @note We catch only the processing errors as any other are considered critical (exceptions)
-      #   and should require a client restart with a backoff
-      rescue Kafka::ProcessingError => e
-        # If there was an error during consumption, we have to log it, pause current partition
-        # and process other things
-        Karafka.monitor.instrument(
-          'connection.client.fetch_loop.error',
-          caller: self,
-          error: e.cause
-        )
-        pause(e.topic, e.partition)
-        retry
+
+        @buffer
+      end
+
+      # Stores offset for a given partition of a given topic based on the provided message.
+      #
+      # @param message [Karafka::Messages::Message]
+      def store_offset(message)
+        @mutex.synchronize do
+          @offsetting = true
+          @kafka.store_offset(message)
+        end
+      end
+
+      # Commits the offset on a current consumer in a non-blocking or blocking way.
+      # Ignoring a case where there would not be an offset (for example when rebalance occurs).
+      #
+      # @param async [Boolean] should the commit happen async or sync (async by default)
+      # @note This will commit all the offsets for the whole consumer. In order to achieve
+      #   granular control over where the offset should be for particular topic partitions, the
+      #   store_offset should be used to only store new offset when we want to to be flushed
+      def commit_offsets(async: true)
+        @mutex.lock
+
+        return unless @offsetting
+
+        @kafka.commit(nil, async)
+        @offsetting = false
+      rescue Rdkafka::RdkafkaError => e
+        return if e.code == :no_offset
+
+        raise e
+      ensure
+        @mutex.unlock
+      end
+
+      # Commits offset in a synchronous way.
+      #
+      # @see `#commit_offset` for more details
+      def commit_offsets!
+        commit_offsets(async: false)
+      end
+
+      # Seek to a particular message. The next poll on the topic/partition will return the
+      # message at the given offset.
+      #
+      # @param message [Messages::Message, Messages::Seek] message to which we want to seek to
+      def seek(message)
+        @kafka.seek(message)
+      end
+
+      # Pauses given partition and moves back to last successful offset processed.
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition
+      # @param offset [Integer] offset of the message on which we want to pause (this message will
+      #   be reprocessed after getting back to processing)
+      # @note This will pause indefinitely and requires manual `#resume`
+      def pause(topic, partition, offset)
+        @mutex.lock
+
+        # Do not pause if the client got closed, would not change anything
+        return if @closed
+
+        tpl = topic_partition_list(topic, partition)
+
+        return unless tpl
+
+        @kafka.pause(tpl)
+
+        pause_msg = Messages::Seek.new(topic, partition, offset)
+
+        seek(pause_msg)
+      ensure
+        @mutex.unlock
       end
 
-      # Gracefully stops topic consumption
+      # Resumes processing of a give topic partition after it was paused.
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition
+      def resume(topic, partition)
+        @mutex.lock
+
+        return if @closed
+
+        tpl = topic_partition_list(topic, partition)
+
+        return unless tpl
+
+        @kafka.resume(tpl)
+      ensure
+        @mutex.unlock
+      end
+
+      # Gracefully stops topic consumption.
+      #
       # @note Stopping running consumers without a really important reason is not recommended
       #   as until all the consumers are stopped, the server will keep running serving only
       #   part of the messages
       def stop
-        @kafka_consumer&.stop
-        @kafka_consumer = nil
+        close
       end
 
-      # Pauses fetching and consumption of a given topic partition
-      # @param topic [String] topic that we want to pause
-      # @param partition [Integer] number partition that we want to pause
-      def pause(topic, partition)
-        args, kwargs = ApiAdapter.pause(topic, partition, consumer_group).values_at(:args, :kwargs)
-        kafka_consumer.pause(*args, **kwargs)
+      # Marks given message as consumed.
+      #
+      # @param [Karafka::Messages::Message] message that we want to mark as processed
+      # @note This method won't trigger automatic offsets commits, rather relying on the offset
+      #   check-pointing trigger that happens with each batch processed
+      def mark_as_consumed(message)
+        store_offset(message)
       end
 
-      # Marks given message as consumed
-      # @param [Karafka::Params::Params] params message that we want to mark as processed
-      # @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
-      #   offsets time-interval based committing
-      def mark_as_consumed(params)
-        kafka_consumer.mark_message_as_processed(
-          *ApiAdapter.mark_message_as_processed(params)
-        )
+      # Marks a given message as consumed and commits the offsets in a blocking way.
+      #
+      # @param [Karafka::Messages::Message] message that we want to mark as processed
+      def mark_as_consumed!(message)
+        mark_as_consumed(message)
+        commit_offsets!
       end
 
-      # Marks a given message as consumed and commit the offsets in a blocking way
-      # @param [Karafka::Params::Params] params message that we want to mark as processed
-      # @note This method commits the offset for each manual marking to be sure
-      #   that offset commit happen asap in case of a crash
-      def mark_as_consumed!(params)
-        mark_as_consumed(params)
-        # Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
-        # before the automatic triggers have kicked in.
-        kafka_consumer.commit_offsets
+      # Closes and resets the client completely.
+      def reset
+        close
+
+        @mutex.synchronize do
+          @closed = false
+          @offsetting = false
+          @kafka = build_consumer
+        end
       end
 
       private
 
-      attr_reader :consumer_group
-
-      # @return [Kafka::Consumer] returns a ready to consume Kafka consumer
-      #   that is set up to consume from topics of a given consumer group
-      def kafka_consumer
-        # @note We don't cache the connection internally because we cache kafka_consumer that uses
-        #   kafka client object instance
-        @kafka_consumer ||= Builder.call(consumer_group).consumer(
-          **ApiAdapter.consumer(consumer_group)
-        ).tap do |consumer|
-          consumer_group.topics.each do |topic|
-            settings = ApiAdapter.subscribe(topic)
-
-            consumer.subscribe(settings[0], **settings[1])
-          end
+      # Commits the stored offsets in a sync way and closes the consumer.
+      def close
+        commit_offsets!
+
+        @mutex.synchronize do
+          @closed = true
+
+          # Remove callbacks runners that were registered
+          ::Karafka::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+          ::Karafka::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+          @kafka.close
         end
-      rescue Kafka::ConnectionError
-        # If we would not wait it will spam log file with failed
-        # attempts if Kafka is down
-        sleep(consumer_group.reconnect_timeout)
-        # We don't log and just re-raise - this will be logged
-        # down the road
-        raise
+      end
+
+      # @param topic [String]
+      # @param partition [Integer]
+      # @return [Rdkafka::Consumer::TopicPartitionList]
+      def topic_partition_list(topic, partition)
+        rdkafka_partition = @kafka
+                            .assignment
+                            .to_h[topic]
+                            &.detect { |part| part.partition == partition }
+
+        return unless rdkafka_partition
+
+        Rdkafka::Consumer::TopicPartitionList.new({ topic => [rdkafka_partition] })
+      end
+
+      # Performs a single poll operation.
+      #
+      # @param timeout [Integer] timeout for a single poll
+      # @return [Array<Rdkafka::Consumer::Message>, nil] fetched messages or nil if nothing polled
+      def poll(timeout)
+        time_poll ||= TimeTrackers::Poll.new(timeout)
+
+        return nil if time_poll.exceeded?
+
+        time_poll.start
+
+        @kafka.poll(time_poll.remaining)
+      rescue ::Rdkafka::RdkafkaError => e
+        raise if time_poll.attempts > MAX_POLL_RETRIES
+        raise unless time_poll.retryable?
+
+        case e.code
+        when :max_poll_exceeded # -147
+          reset
+        when :transport # -195
+          reset
+        when :rebalance_in_progress # -27
+          reset
+        when :not_coordinator # 16
+          reset
+        when :network_exception # 13
+          reset
+        end
+
+        time_poll.checkpoint
+
+        raise unless time_poll.retryable?
+
+        time_poll.backoff
+
+        retry
+      end
+
+      # Builds a new rdkafka consumer instance based on the subscription group configuration
+      # @return [Rdkafka::Consumer]
+      def build_consumer
+        ::Rdkafka::Config.logger = ::Karafka::App.config.logger
+        config = ::Rdkafka::Config.new(@subscription_group.kafka)
+        config.consumer_rebalance_listener = @rebalance_manager
+        consumer = config.consumer
+        consumer.subscribe(*@subscription_group.topics.map(&:name))
+        @name = consumer.name
+
+        # Register statistics runner for this particular type of callbacks
+        ::Karafka::Instrumentation.statistics_callbacks.add(
+          @subscription_group.id,
+          Instrumentation::Callbacks::Statistics.new(
+            @subscription_group.id,
+            @subscription_group.consumer_group_id,
+            @name,
+            ::Karafka::App.config.monitor
+          )
+        )
+
+        # Register error tracking callback
+        ::Karafka::Instrumentation.error_callbacks.add(
+          @subscription_group.id,
+          Instrumentation::Callbacks::Error.new(
+            @subscription_group.id,
+            @subscription_group.consumer_group_id,
+            @name,
+            ::Karafka::App.config.monitor
+          )
+        )
+
+        consumer
       end
     end
   end

data/lib/karafka/connection/listener.rb

@@ -2,69 +2,168 @@
 
 module Karafka
   module Connection
-    # A single listener that listens to incoming messages from a single route
-    # @note It does not loop on itself - it needs to be executed in a loop
-    # @note Listener itself does nothing with the message - it will return to the block
-    #   a raw Kafka::FetchedMessage
+    # A single listener that listens to incoming messages from a single subscription group.
+    # It polls the messages and then enqueues. It also takes care of potential recovery from
+    # critical errors by restarting everything in a safe manner.
     class Listener
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
-      #   on what topics and with what settings should we listen
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
+      def initialize(subscription_group, jobs_queue)
+        @subscription_group = subscription_group
+        @jobs_queue = jobs_queue
+        @pauses_manager = PausesManager.new
+        @client = Client.new(@subscription_group)
+        @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
       end
 
-      # Runs prefetch callbacks and executes the main listener fetch loop
+      # Runs the main listener fetch loop.
+      #
+      # @note Prefetch callbacks can be used to seek offset or do other things before we actually
+      #   start consuming data
       def call
         Karafka.monitor.instrument(
           'connection.listener.before_fetch_loop',
-          consumer_group: @consumer_group,
-          client: client
+          caller: self,
+          subscription_group: @subscription_group,
+          client: @client
         )
+
         fetch_loop
       end
 
       private
 
-      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # Fetches the data and adds it to the jobs queue.
+      #
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
-      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
-      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
+      #   Since it is run inside Karafka::Connection::Runner thread - catching all the exceptions
+      #   won't crash the whole process. Here we mostly focus on catching the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
-      #   propagate this far
+      #   propagate this far.
       def fetch_loop
-        # @note What happens here is a delegation of processing to a proper processor based
-        #   on the incoming messages characteristics
-        client.fetch_loop do |raw_data, type|
-          Karafka.monitor.instrument('connection.listener.fetch_loop')
-
-          case type
-          when :message
-            MessageDelegator.call(@consumer_group.id, raw_data)
-          when :batch
-            BatchDelegator.call(@consumer_group.id, raw_data)
-          end
+        until Karafka::App.stopping?
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop',
+            caller: self,
+            client: @client
+          )
+
+          resume_paused_partitions
+          # We need to fetch data before we revoke lost partitions details as during the polling
+          # the callbacks for tracking lost partitions are triggered. Otherwise we would be always
+          # one batch behind.
+          messages_buffer = @client.batch_poll
+
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop.received',
+            caller: self,
+            messages_buffer: messages_buffer
+          )
+
+          # If there were revoked partitions, we need to wait on their jobs to finish before
+          # distributing consuming jobs as upon revoking, we might get assigned to the same
+          # partitions, thus getting their jobs. The revoking jobs need to finish before
+          # appropriate consumers are taken down and re-created
+          wait(@subscription_group) if distribute_revoke_lost_partitions_jobs
+
+          distribute_partitions_jobs(messages_buffer)
+
+          # We wait only on jobs from our subscription group. Other groups are independent.
+          wait(@subscription_group)
+
+          # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
+          # if needed by using manual offset management.
+          @client.commit_offsets
         end
+
+        shutdown
+
         # This is on purpose - see the notes for this method
         # rubocop:disable Lint/RescueException
       rescue Exception => e
-        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable Lint/RescueException
-        # We can stop client without a problem, as it will reinitialize itself when running the
-        # `fetch_loop` again
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'connection.listener.fetch_loop.error'
+        )
+
+        restart
+
+        sleep(1) && retry
+      end
+
+      # Resumes processing of partitions that were paused due to an error.
+      def resume_paused_partitions
+        @pauses_manager.resume { |topic, partition| @client.resume(topic, partition) }
+      end
+
+      # Enqueues revoking jobs for partitions that were taken away from the running process.
+      # @return [Boolean] was there anything to revoke
+      def distribute_revoke_lost_partitions_jobs
+        revoked_partitions = @client.rebalance_manager.revoked_partitions
+
+        return false if revoked_partitions.empty?
+
+        revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            pause = @pauses_manager.fetch(topic, partition)
+            executor = @executors.fetch(topic, partition, pause)
+            @jobs_queue << Processing::Jobs::Revoked.new(executor)
+          end
+        end
+
+        true
+      end
+
+      # Takes the messages per topic partition and enqueues processing jobs in threads.
+      #
+      # @param messages_buffer [Karafka::Connection::MessagesBuffer] buffer with messages
+      def distribute_partitions_jobs(messages_buffer)
+        messages_buffer.each do |topic, partition, messages|
+          pause = @pauses_manager.fetch(topic, partition)
+
+          next if pause.paused?
+
+          executor = @executors.fetch(topic, partition, pause)
+
+          @jobs_queue << Processing::Jobs::Consume.new(executor, messages)
+        end
+      end
+
+      # Waits for all the jobs from a given subscription group to finish before moving forward
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      def wait(subscription_group)
+        @jobs_queue.wait(subscription_group.id)
+      end
+
+      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
+      # stops kafka client.
+      def shutdown
+        @jobs_queue.close
+        # This runs synchronously, making sure we finish all the shutdowns before we stop the
+        # client.
+        @executors.shutdown
+        @client.commit_offsets!
         @client.stop
-        # We need to clear the consumers cache for current connection when fatal error happens and
-        # we reset the connection. Otherwise for consumers with manual offset management, the
-        # persistence might have stored some data that would be reprocessed
-        Karafka::Persistence::Consumers.clear
-        sleep(@consumer_group.reconnect_timeout) && retry
       end
 
-      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
-      #   consumption
-      def client
-        @client ||= Client.new(@consumer_group)
+      # We can stop client without a problem, as it will reinitialize itself when running the
+      # `#fetch_loop` again. We just need to remember to also reset the runner as it is a long
+      # running one, so with a new connection to Kafka, we need to initialize the state of the
+      # runner and underlying consumers once again.
+      def restart
+        # If there was any problem with processing, before we reset things we need to make sure,
+        # there are no jobs in the queue. Otherwise it could lead to leakage in between client
+        # resetting.
+        @jobs_queue.wait(@subscription_group.id)
+        @jobs_queue.clear(@subscription_group.id)
+        @client.reset
+        @pauses_manager = PausesManager.new
+        @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end
   end

data/lib/karafka/connection/messages_buffer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer for messages.
+    # When message is added to this buffer, it gets assigned to an array with other messages from
+    # the same topic and partition.
+    #
+    # @note This buffer is NOT threadsafe.
+    class MessagesBuffer
+      attr_reader :size
+
+      # @return [Karafka::Connection::MessagesBuffer] buffer instance
+      def initialize
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Iterates over aggregated data providing messages per topic partition.
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # Adds a message to the buffer.
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
+      def <<(message)
+        @size += 1
+        @groups[message.topic][message.partition] << message
+      end
+
+      # Removes all the data from the buffer.
+      #
+      # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
+      #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
+      #   may be used in other threads for data processing, thus if we would clear it, we could
+      #   potentially clear a raw messages array for a job that is in the jobs queue.
+      def clear
+        @size = 0
+        @groups.each_value(&:clear)
+      end
+    end
+  end
+end