karafka 1.4.0 → 2.0.10

data/lib/karafka/connection/listener.rb

@@ -2,69 +2,281 @@
 
 module Karafka
   module Connection
-    # A single listener that listens to incoming messages from a single route
-    # @note It does not loop on itself - it needs to be executed in a loop
-    # @note Listener itself does nothing with the message - it will return to the block
-    #   a raw Kafka::FetchedMessage
+    # A single listener that listens to incoming messages from a single subscription group.
+    # It polls the messages and then enqueues jobs. It also takes care of potential recovery from
+    # critical errors by restarting everything in a safe manner.
+    #
+    # This is the heart of the consumption process.
     class Listener
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
-      #   on what topics and with what settings should we listen
+      include Helpers::Async
+
+      # Can be useful for logging
+      # @return [String] id of this listener
+      attr_reader :id
+
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
+      def initialize(subscription_group, jobs_queue)
+        proc_config = ::Karafka::App.config.internal.processing
+
+        @id = SecureRandom.uuid
+        @subscription_group = subscription_group
+        @jobs_queue = jobs_queue
+        @coordinators = Processing::CoordinatorsBuffer.new
+        @client = Client.new(@subscription_group)
+        @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
+        @jobs_builder = proc_config.jobs_builder
+        @partitioner = proc_config.partitioner_class.new(subscription_group)
+        # We reference scheduler here as it is much faster than fetching this each time
+        @scheduler = proc_config.scheduler
+        # We keep one buffer for messages to preserve memory and not allocate extra objects
+        # We can do this that way because we always first schedule jobs using messages before we
+        # fetch another batch.
+        @messages_buffer = MessagesBuffer.new(subscription_group)
+        @mutex = Mutex.new
+        @stopped = false
       end
 
-      # Runs prefetch callbacks and executes the main listener fetch loop
+      # Runs the main listener fetch loop.
+      #
+      # @note Prefetch callbacks can be used to seek offset or do other things before we actually
+      #   start consuming data
       def call
         Karafka.monitor.instrument(
           'connection.listener.before_fetch_loop',
-          consumer_group: @consumer_group,
-          client: client
+          caller: self,
+          subscription_group: @subscription_group,
+          client: @client
         )
+
         fetch_loop
       end
 
+      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
+      # stops kafka client.
+      #
+      # @note This method is not private despite being part of the fetch loop because in case of
+      #   a forceful shutdown, it may be invoked from a separate thread
+      #
+      # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
+      def shutdown
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.commit_offsets!
+          @client.stop
+        end
+      end
+
       private
 
-      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # Fetches the data and adds it to the jobs queue.
+      #
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
-      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
-      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
+      #   Since it is run inside Karafka::Connection::Runner thread - catching all the exceptions
+      #   won't crash the whole process. Here we mostly focus on catching the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
-      #   propagate this far
+      #   propagate this far.
       def fetch_loop
-        # @note What happens here is a delegation of processing to a proper processor based
-        #   on the incoming messages characteristics
-        client.fetch_loop do |raw_data, type|
-          Karafka.monitor.instrument('connection.listener.fetch_loop')
-
-          case type
-          when :message
-            MessageDelegator.call(@consumer_group.id, raw_data)
-          when :batch
-            BatchDelegator.call(@consumer_group.id, raw_data)
+        until Karafka::App.stopping?
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop',
+            caller: self,
+            client: @client
+          )
+
+          resume_paused_partitions
+
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop.received',
+            caller: self,
+            messages_buffer: @messages_buffer
+          ) do
+            # We need to fetch data before we revoke lost partitions details as during the polling
+            # the callbacks for tracking lost partitions are triggered. Otherwise we would be
+            # always one batch behind.
+            poll_and_remap_messages
           end
+
+          # If there were revoked partitions, we need to wait on their jobs to finish before
+          # distributing consuming jobs as upon revoking, we might get assigned to the same
+          # partitions, thus getting their jobs. The revoking jobs need to finish before
+          # appropriate consumers are taken down and re-created
+          build_and_schedule_revoke_lost_partitions_jobs
+
+          # We wait only on jobs from our subscription group. Other groups are independent.
+          # This will block on revoked jobs until they are finished. Those are not meant to last
+          # long and should not have any bigger impact on the system. Doing this in a blocking way
+          # simplifies the overall design and prevents from race conditions
+          wait
+
+          build_and_schedule_consumption_jobs
+
+          wait
+
+          # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
+          # if needed by using manual offset management.
+          @client.commit_offsets
         end
+
+        # If we are stopping we will no longer schedule any jobs despite polling.
+        # We need to keep polling not to exceed the `max.poll.interval` for long-running
+        # non-blocking jobs and we need to allow them to finish. We however do not want to
+        # enqueue any new jobs. It's worth keeping in mind that it is the end user responsibility
+        # to detect shutdown in their long-running logic or else Karafka will force shutdown
+        # after a while.
+        #
+        # We do not care about resuming any partitions or lost jobs as we do not plan to do
+        # anything with them as we're in the shutdown phase.
+        wait_with_poll
+
+        # We do not want to schedule the shutdown jobs prior to finishing all the jobs
+        # (including non-blocking) as there might be a long-running job with a shutdown and then
+        # we would run two jobs in parallel for the same executor and consumer. We do not want that
+        # as it could create a race-condition.
+        build_and_schedule_shutdown_jobs
+
+        wait_with_poll
+
+        shutdown
+
         # This is on purpose - see the notes for this method
         # rubocop:disable Lint/RescueException
       rescue Exception => e
-        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable Lint/RescueException
-        # We can stop client without a problem, as it will reinitialize itself when running the
-        # `fetch_loop` again
-        @client.stop
-        # We need to clear the consumers cache for current connection when fatal error happens and
-        # we reset the connection. Otherwise for consumers with manual offset management, the
-        # persistence might have stored some data that would be reprocessed
-        Karafka::Persistence::Consumers.clear
-        sleep(@consumer_group.reconnect_timeout) && retry
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'connection.listener.fetch_loop.error'
+        )
+
+        restart
+
+        sleep(1) && retry
+      end
+
+      # Resumes processing of partitions that were paused due to an error.
+      def resume_paused_partitions
+        @coordinators.resume do |topic, partition|
+          @client.resume(topic, partition)
+        end
+      end
+
+      # Enqueues revoking jobs for partitions that were taken away from the running process.
+      def build_and_schedule_revoke_lost_partitions_jobs
+        revoked_partitions = @client.rebalance_manager.revoked_partitions
+
+        # Stop early to save on some execution and array allocation
+        return if revoked_partitions.empty?
+
+        jobs = []
+
+        revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @coordinators.revoke(topic, partition)
+
+            # There may be a case where we have lost partition of which data we have never
+            # processed (if it was assigned and revoked really fast), thus we may not have it
+            # here. In cases like this, we do not run a revocation job
+            @executors.find_all(topic, partition).each do |executor|
+              job = @jobs_builder.revoked(executor)
+              job.before_enqueue
+              jobs << job
+            end
+
+            # We need to remove all the executors of a given topic partition that we have lost, so
+            # next time we pick up it's work, new executors kick in. This may be needed especially
+            # for LRJ where we could end up with a race condition
+            # This revocation needs to happen after the jobs are scheduled, otherwise they would
+            # be scheduled with new executors instead of old
+            @executors.revoke(topic, partition)
+          end
+        end
+
+        @scheduler.schedule_revocation(@jobs_queue, jobs)
+      end
+
+      # Enqueues the shutdown jobs for all the executors that exist in our subscription group
+      def build_and_schedule_shutdown_jobs
+        jobs = []
+
+        @executors.each do |_, _, executor|
+          job = @jobs_builder.shutdown(executor)
+          job.before_enqueue
+          jobs << job
+        end
+
+        @scheduler.schedule_shutdown(@jobs_queue, jobs)
+      end
+
+      # Polls messages within the time and amount boundaries defined in the settings and then
+      # builds karafka messages based on the raw rdkafka messages buffer returned by the
+      # `#batch_poll` method.
+      #
+      # @note There are two buffers, one for raw messages and one for "built" karafka messages
+      def poll_and_remap_messages
+        @messages_buffer.remap(
+          @client.batch_poll
+        )
+      end
+
+      # Takes the messages per topic partition and enqueues processing jobs in threads using
+      # given scheduler.
+      def build_and_schedule_consumption_jobs
+        return if @messages_buffer.empty?
+
+        jobs = []
+
+        @messages_buffer.each do |topic, partition, messages|
+          coordinator = @coordinators.find_or_create(topic, partition)
+
+          # Start work coordination for this topic partition
+          coordinator.start(messages)
+
+          @partitioner.call(topic, messages) do |group_id, partition_messages|
+            # Count the job we're going to create here
+            coordinator.increment
+            executor = @executors.find_or_create(topic, partition, group_id)
+            job = @jobs_builder.consume(executor, partition_messages, coordinator)
+            job.before_enqueue
+            jobs << job
+          end
+        end
+
+        @scheduler.schedule_consumption(@jobs_queue, jobs)
+      end
+
+      # Waits for all the jobs from a given subscription group to finish before moving forward
+      def wait
+        @jobs_queue.wait(@subscription_group.id)
+      end
+
+      # Waits without blocking the polling
+      # This should be used only when we no longer plan to use any incoming data and we can safely
+      # discard it
+      def wait_with_poll
+        @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
       end
 
-      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
-      #   consumption
-      def client
-        @client ||= Client.new(@consumer_group)
+      # We can stop client without a problem, as it will reinitialize itself when running the
+      # `#fetch_loop` again. We just need to remember to also reset the runner as it is a long
+      # running one, so with a new connection to Kafka, we need to initialize the state of the
+      # runner and underlying consumers once again.
+      def restart
+        # If there was any problem with processing, before we reset things we need to make sure,
+        # there are no jobs in the queue. Otherwise it could lead to leakage in between client
+        # resetting.
+        @jobs_queue.wait(@subscription_group.id)
+        @jobs_queue.clear(@subscription_group.id)
+        @client.reset
+        @coordinators.reset
+        @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end
   end

data/lib/karafka/connection/listeners_batch.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Abstraction layer around listeners batch.
+    class ListenersBatch
+      include Enumerable
+
+      # @param jobs_queue [JobsQueue]
+      # @return [ListenersBatch]
+      def initialize(jobs_queue)
+        @batch = App.subscription_groups.map do |subscription_group|
+          Connection::Listener.new(subscription_group, jobs_queue)
+        end
+      end
+
+      # Iterates over available listeners and yields each listener
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+    end
+  end
+end

data/lib/karafka/connection/messages_buffer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer used to build and store karafka messages built based on raw librdkafka messages.
+    #
+    # Why do we have two buffers? `RawMessagesBuffer` is used to store raw messages and to handle
+    #   cases related to partition revocation and reconnections. It is "internal" to the listening
+    #   process. `MessagesBuffer` on the other hand is used to "translate" those raw messages that
+    #   we know that are ok into Karafka messages and to simplify further work with them.
+    #
+    # While it adds a bit of overhead, it makes conceptual things much easier and it adds only two
+    #   simple hash iterations over messages batch.
+    #
+    # @note This buffer is NOT thread safe. We do not worry about it as we do not use it outside
+    #   of the main listener loop. It can be cleared after the jobs are scheduled with messages
+    #   it stores, because messages arrays are not "cleared" in any way directly and their
+    #   reference stays.
+    class MessagesBuffer
+      attr_reader :size
+
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Remaps raw messages from the raw messages buffer to Karafka messages
+      # @param raw_messages_buffer [RawMessagesBuffer] buffer with raw messages
+      def remap(raw_messages_buffer)
+        clear unless @size.zero?
+
+        # Since it happens "right after" we've received the messages, it is close enough it time
+        # to be used as the moment we received messages.
+        received_at = Time.now
+
+        raw_messages_buffer.each do |topic, partition, messages|
+          @size += messages.count
+
+          ktopic = @subscription_group.topics.find(topic)
+
+          @groups[topic][partition] = messages.map do |message|
+            Messages::Builders::Message.call(
+              message,
+              ktopic,
+              received_at
+            )
+          end
+        end
+      end
+
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # @return [Boolean] is the buffer empty or does it contain any messages
+      def empty?
+        @size.zero?
+      end
+
+      private
+
+      # Clears the buffer completely
+      def clear
+        @size = 0
+        @groups.clear
+      end
+    end
+  end
+end

data/lib/karafka/connection/pauses_manager.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Partitions pauses management abstraction layer.
+    # It aggregates all the pauses for all the partitions that we're working with.
+    class PausesManager
+      # @return [Karafka::Connection::PausesManager] pauses manager
+      def initialize
+        @pauses = Hash.new do |h, k|
+          h[k] = {}
+        end
+      end
+
+      # Creates or fetches pause tracker of a given topic partition.
+      #
+      # @param topic [String] topic name
+      # @param partition [Integer] partition number
+      # @return [Karafka::TimeTrackers::Pause] pause tracker instance
+      def fetch(topic, partition)
+        @pauses[topic][partition] ||= TimeTrackers::Pause.new(
+          timeout: Karafka::App.config.pause_timeout,
+          max_timeout: Karafka::App.config.pause_max_timeout,
+          exponential_backoff: Karafka::App.config.pause_with_exponential_backoff
+        )
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      def resume
+        @pauses.each do |topic, partitions|
+          partitions.each do |partition, pause|
+            next unless pause.paused?
+            next unless pause.expired?
+
+            pause.resume
+
+            yield(topic, partition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer for raw librdkafka messages.
+    #
+    # When message is added to this buffer, it gets assigned to an array with other messages from
+    # the same topic and partition.
+    #
+    # @note This buffer is NOT threadsafe.
+    #
+    # @note We store data here in groups per topic partition to handle the revocation case, where
+    #    we may need to remove messages from a single topic partition.
+    class RawMessagesBuffer
+      attr_reader :size
+
+      # @return [Karafka::Connection::MessagesBuffer] buffer instance
+      def initialize
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Adds a message to the buffer.
+      #
+      # @param message [Rdkafka::Consumer::Message] raw rdkafka message
+      # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
+      def <<(message)
+        @size += 1
+        @groups[message.topic][message.partition] << message
+      end
+
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # Removes given topic and partition data out of the buffer
+      # This is used when there's a partition revocation
+      # @param topic [String] topic we're interested in
+      # @param partition [Integer] partition of which data we want to remove
+      def delete(topic, partition)
+        return unless @groups.key?(topic)
+        return unless @groups.fetch(topic).key?(partition)
+
+        topic_data = @groups.fetch(topic)
+        topic_data.delete(partition)
+
+        recount!
+
+        # If there are no more partitions to handle in a given topic, remove it completely
+        @groups.delete(topic) if topic_data.empty?
+      end
+
+      # Removes duplicated messages from the same partitions
+      # This should be used only when rebalance occurs, as we may get data again we already have
+      # due to the processing from the last offset. In cases like this, we may get same data
+      # again and we do want to ensure as few duplications as possible
+      def uniq!
+        @groups.each_value do |partitions|
+          partitions.each_value do |messages|
+            messages.uniq!(&:offset)
+          end
+        end
+
+        recount!
+      end
+
+      # Removes all the data from the buffer.
+      #
+      # @note We do not clear the whole groups hash but rather we clear the partition hashes, so
+      #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
+      #   may be used in other threads for data processing, thus if we would clear it, we could
+      #   potentially clear a raw messages array for a job that is in the jobs queue.
+      def clear
+        @size = 0
+        @groups.each_value(&:clear)
+      end
+
+      private
+
+      # Updates the messages count if we performed any operations that could change the state
+      def recount!
+        @size = @groups.each_value.sum do |partitions|
+          partitions.each_value.map(&:count).sum
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/rebalance_manager.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Manager for tracking changes in the partitions assignment.
+    #
+    # We need tracking of those to clean up consumers that will no longer process given partitions
+    # as they were taken away.
+    #
+    # @note Since this does not happen really often, we try to stick with same objects for the
+    #   empty states most of the time, so we don't create many objects during the manager life
+    #
+    # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
+    #   that are lost, are those that got revoked but did not get re-assigned back. We do not
+    #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
+    #   from the rest of the code perspective as only those that are lost are truly revoked.
+    class RebalanceManager
+      # Empty array for internal usage not to create new objects
+      EMPTY_ARRAY = [].freeze
+
+      attr_reader :assigned_partitions, :revoked_partitions
+
+      private_constant :EMPTY_ARRAY
+
+      # @return [RebalanceManager]
+      def initialize
+        @assigned_partitions = {}
+        @revoked_partitions = {}
+        @changed = false
+      end
+
+      # Resets the rebalance manager state
+      # This needs to be done before each polling loop as during the polling, the state may be
+      # changed
+      def clear
+        @assigned_partitions.clear
+        @revoked_partitions.clear
+        @changed = false
+      end
+
+      # @return [Boolean] indicates a state change in the partitions assignment
+      def changed?
+        @changed
+      end
+
+      # Callback that kicks in inside of rdkafka, when new partitions are assigned.
+      #
+      # @private
+      # @param _ [Rdkafka::Consumer]
+      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
+      def on_partitions_assigned(_, partitions)
+        @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # Callback that kicks in inside of rdkafka, when partitions are revoked.
+      #
+      # @private
+      # @param _ [Rdkafka::Consumer]
+      # @param partitions [Rdkafka::Consumer::TopicPartitionList]
+      def on_partitions_revoked(_, partitions)
+        @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
+        @changed = true
+      end
+
+      # We consider as lost only partitions that were taken away and not re-assigned back to us
+      def lost_partitions
+        lost_partitions = {}
+
+        revoked_partitions.each do |topic, partitions|
+          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
+        end
+
+        lost_partitions
+      end
+    end
+  end
+end

data/lib/karafka/contracts/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Base contract for all Karafka contracts
+    class Base < ::Karafka::Core::Contractable::Contract
+      # @param data [Hash] data for validation
+      # @return [Boolean] true if all good
+      # @raise [Errors::InvalidConfigurationError] invalid configuration error
+      # @note We use contracts only in the config validation context, so no need to add support
+      #   for multiple error classes. It will be added when it will be needed.
+      def validate!(data)
+        super(data, Errors::InvalidConfigurationError)
+      end
+    end
+  end
+end