karafka 1.4.12 → 2.2.10

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # This object represents a collective status of execution of group of listeners running inside
+    # of one consumer group but in separate subscription groups.
+    #
+    # There are cases when we do not want to close a given client when others from the same
+    # consumer group are running because it can cause instabilities due to early shutdown of some
+    # of the clients out of same consumer group.
+    #
+    # We also want to make sure, we close one consumer at a time while others can continue polling.
+    #
+    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
+    # without having a chance to commit changes.
+    class ConsumerGroupCoordinator
+      # @param group_size [Integer] number of separate subscription groups in a consumer group
+      def initialize(group_size)
+        @shutdown_lock = Mutex.new
+        @group_size = group_size
+        @finished = Set.new
+      end
+
+      # @return [Boolean] true if all the subscription groups from a given consumer group are
+      #   finished
+      def finished?
+        @finished.size == @group_size
+      end
+
+      # @return [Boolean] can we start shutdown on a given listener
+      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
+      def shutdown?
+        finished? && @shutdown_lock.try_lock
+      end
+
+      # Unlocks the shutdown lock
+      def unlock
+        @shutdown_lock.unlock if @shutdown_lock.owned?
+      end
+
+      # Marks given listener as finished
+      # @param listener_id [String]
+      def finish_work(listener_id)
+        @finished << listener_id
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -2,69 +2,325 @@
 
 module Karafka
   module Connection
-    # A single listener that listens to incoming messages from a single route
-    # @note It does not loop on itself - it needs to be executed in a loop
-    # @note Listener itself does nothing with the message - it will return to the block
-    #   a raw Kafka::FetchedMessage
+    # A single listener that listens to incoming messages from a single subscription group.
+    # It polls the messages and then enqueues jobs. It also takes care of potential recovery from
+    # critical errors by restarting everything in a safe manner.
+    #
+    # This is the heart of the consumption process.
     class Listener
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] consumer group that holds details
-      #   on what topics and with what settings should we listen
+      include Helpers::Async
+
+      # Can be useful for logging
+      # @return [String] id of this listener
+      attr_reader :id
+
+      # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(consumer_group)
-        @consumer_group = consumer_group
+      def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
+        proc_config = ::Karafka::App.config.internal.processing
+
+        @id = SecureRandom.hex(6)
+        @consumer_group_coordinator = consumer_group_coordinator
+        @subscription_group = subscription_group
+        @jobs_queue = jobs_queue
+        @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
+        @client = Client.new(@subscription_group)
+        @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
+        @jobs_builder = proc_config.jobs_builder
+        @partitioner = proc_config.partitioner_class.new(subscription_group)
+        # We reference scheduler here as it is much faster than fetching this each time
+        @scheduler = proc_config.scheduler
+        # We keep one buffer for messages to preserve memory and not allocate extra objects
+        # We can do this that way because we always first schedule jobs using messages before we
+        # fetch another batch.
+        @messages_buffer = MessagesBuffer.new(subscription_group)
+        @mutex = Mutex.new
+        @stopped = false
       end
 
-      # Runs prefetch callbacks and executes the main listener fetch loop
+      # Runs the main listener fetch loop.
+      #
+      # @note Prefetch callbacks can be used to seek offset or do other things before we actually
+      #   start consuming data
       def call
         Karafka.monitor.instrument(
           'connection.listener.before_fetch_loop',
-          consumer_group: @consumer_group,
-          client: client
+          caller: self,
+          client: @client,
+          subscription_group: @subscription_group
         )
+
         fetch_loop
       end
 
+      # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
+      # stops kafka client.
+      #
+      # @note This method is not private despite being part of the fetch loop because in case of
+      #   a forceful shutdown, it may be invoked from a separate thread
+      #
+      # @note We wrap it with a mutex exactly because of the above case of forceful shutdown
+      def shutdown
+        return if @stopped
+
+        @mutex.synchronize do
+          @stopped = true
+          @executors.clear
+          @coordinators.reset
+          @client.stop
+        end
+      end
+
       private
 
-      # Opens connection, gets messages and calls a block for each of the incoming messages
+      # Fetches the data and adds it to the jobs queue.
+      #
       # @note We catch all the errors here, so they don't affect other listeners (or this one)
       #   so we will be able to listen and consume other incoming messages.
-      #   Since it is run inside Karafka::Connection::ActorCluster - catching all the exceptions
-      #   won't crash the whole cluster. Here we mostly focus on catching the exceptions related to
+      #   Since it is run inside Karafka::Connection::Runner thread - catching all the exceptions
+      #   won't crash the whole process. Here we mostly focus on catching the exceptions related to
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
-      #   propagate this far
+      #   propagate this far.
       def fetch_loop
-        # @note What happens here is a delegation of processing to a proper processor based
-        #   on the incoming messages characteristics
-        client.fetch_loop do |raw_data, type|
-          Karafka.monitor.instrument('connection.listener.fetch_loop')
-
-          case type
-          when :message
-            MessageDelegator.call(@consumer_group.id, raw_data)
-          when :batch
-            BatchDelegator.call(@consumer_group.id, raw_data)
+        # Run the main loop as long as we are not stopping or moving into quiet mode
+        until Karafka::App.done?
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop',
+            caller: self,
+            client: @client,
+            subscription_group: @subscription_group
+          )
+
+          resume_paused_partitions
+
+          Karafka.monitor.instrument(
+            'connection.listener.fetch_loop.received',
+            caller: self,
+            client: @client,
+            subscription_group: @subscription_group,
+            messages_buffer: @messages_buffer
+          ) do
+            # We need to fetch data before we revoke lost partitions details as during the polling
+            # the callbacks for tracking lost partitions are triggered. Otherwise we would be
+            # always one batch behind.
+            poll_and_remap_messages
           end
+
+          # If there were revoked partitions, we need to wait on their jobs to finish before
+          # distributing consuming jobs as upon revoking, we might get assigned to the same
+          # partitions, thus getting their jobs. The revoking jobs need to finish before
+          # appropriate consumers are taken down and re-created
+          build_and_schedule_revoked_jobs_for_revoked_partitions
+
+          # We wait only on jobs from our subscription group. Other groups are independent.
+          # This will block on revoked jobs until they are finished. Those are not meant to last
+          # long and should not have any bigger impact on the system. Doing this in a blocking way
+          # simplifies the overall design and prevents from race conditions
+          wait
+
+          build_and_schedule_consumption_jobs
+
+          wait
         end
+
+        # If we are stopping we will no longer schedule any regular jobs despite polling.
+        # We need to keep polling not to exceed the `max.poll.interval` for long-running
+        # non-blocking jobs and we need to allow them to finish. We however do not want to
+        # enqueue any new jobs. It's worth keeping in mind that it is the end user responsibility
+        # to detect shutdown in their long-running logic or else Karafka will force shutdown
+        # after a while.
+        #
+        # We do not care about resuming any partitions or lost jobs as we do not plan to do
+        # anything with them as we're in the shutdown phase.
+        #
+        # What we do care however is the ability to still run revocation jobs in case anything
+        # would change in the cluster. We still want to notify the long-running jobs about changes
+        # that occurred in the cluster.
+        wait_pinging(
+          wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
+          after_ping: -> { build_and_schedule_revoked_jobs_for_revoked_partitions }
+        )
+
+        # We do not want to schedule the shutdown jobs prior to finishing all the jobs
+        # (including non-blocking) as there might be a long-running job with a shutdown and then
+        # we would run two jobs in parallel for the same executor and consumer. We do not want that
+        # as it could create a race-condition.
+        build_and_schedule_shutdown_jobs
+
+        # Wait until all the shutdown jobs are done
+        wait_pinging(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
+
+        # Once all the work is done, we need to decrement counter of active subscription groups
+        # within this consumer group
+        @consumer_group_coordinator.finish_work(id)
+
+        # Wait if we're in the process of finishing started work or finished all the work and
+        # just sitting and being quiet
+        wait_pinging(wait_until: -> { !(Karafka::App.quieting? || Karafka::App.quiet?) })
+
+        # We need to wait until all the work in the whole consumer group (local to the process)
+        # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
+        # warning notifications.
+        wait_pinging(wait_until: -> { @consumer_group_coordinator.shutdown? })
+
+        # This extra ping will make sure we've refreshed the rebalance state after other instances
+        # potentially shutdown. This will prevent us from closing with a dangling callback
+        @client.ping
+
+        shutdown
+
         # This is on purpose - see the notes for this method
         # rubocop:disable Lint/RescueException
       rescue Exception => e
-        Karafka.monitor.instrument('connection.listener.fetch_loop.error', caller: self, error: e)
         # rubocop:enable Lint/RescueException
-        # We can stop client without a problem, as it will reinitialize itself when running the
-        # `fetch_loop` again
-        @client.stop
-        # We need to clear the consumers cache for current connection when fatal error happens and
-        # we reset the connection. Otherwise for consumers with manual offset management, the
-        # persistence might have stored some data that would be reprocessed
-        Karafka::Persistence::Consumers.clear
-        sleep(@consumer_group.reconnect_timeout) && retry
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'connection.listener.fetch_loop.error'
+        )
+
+        restart
+
+        sleep(1) && retry
+      ensure
+        @consumer_group_coordinator.unlock
+      end
+
+      # Resumes processing of partitions that were paused due to an error.
+      def resume_paused_partitions
+        @coordinators.resume do |topic, partition|
+          @client.resume(topic.name, partition)
+        end
+      end
+
+      # Enqueues revoking jobs for partitions that were taken away from the running process.
+      def build_and_schedule_revoked_jobs_for_revoked_partitions
+        revoked_partitions = @client.rebalance_manager.revoked_partitions
+
+        # Stop early to save on some execution and array allocation
+        return if revoked_partitions.empty?
+
+        jobs = []
+
+        revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @coordinators.revoke(topic, partition)
+
+            # There may be a case where we have lost partition of which data we have never
+            # processed (if it was assigned and revoked really fast), thus we may not have it
+            # here. In cases like this, we do not run a revocation job
+            @executors.find_all(topic, partition).each do |executor|
+              job = @jobs_builder.revoked(executor)
+              job.before_enqueue
+              jobs << job
+            end
+
+            # We need to remove all the executors of a given topic partition that we have lost, so
+            # next time we pick up it's work, new executors kick in. This may be needed especially
+            # for LRJ where we could end up with a race condition
+            # This revocation needs to happen after the jobs are scheduled, otherwise they would
+            # be scheduled with new executors instead of old
+            @executors.revoke(topic, partition)
+          end
+        end
+
+        @scheduler.schedule_revocation(@jobs_queue, jobs)
+      end
+
+      # Enqueues the shutdown jobs for all the executors that exist in our subscription group
+      def build_and_schedule_shutdown_jobs
+        jobs = []
+
+        @executors.each do |executor|
+          job = @jobs_builder.shutdown(executor)
+          job.before_enqueue
+          jobs << job
+        end
+
+        @scheduler.schedule_shutdown(@jobs_queue, jobs)
+      end
+
+      # Polls messages within the time and amount boundaries defined in the settings and then
+      # builds karafka messages based on the raw rdkafka messages buffer returned by the
+      # `#batch_poll` method.
+      #
+      # @note There are two buffers, one for raw messages and one for "built" karafka messages
+      def poll_and_remap_messages
+        @messages_buffer.remap(
+          @client.batch_poll
+        )
+      end
+
+      # Takes the messages per topic partition and enqueues processing jobs in threads using
+      # given scheduler.
+      def build_and_schedule_consumption_jobs
+        return if @messages_buffer.empty?
+
+        jobs = []
+
+        @messages_buffer.each do |topic, partition, messages|
+          coordinator = @coordinators.find_or_create(topic, partition)
+          # Start work coordination for this topic partition
+          coordinator.start(messages)
+
+          # We do not increment coordinator for idle job because it's not a user related one
+          # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
+          if messages.empty?
+            executor = @executors.find_or_create(topic, partition, 0, coordinator)
+            jobs << @jobs_builder.idle(executor)
+          else
+            @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+              executor = @executors.find_or_create(topic, partition, group_id, coordinator)
+              coordinator.increment
+              jobs << @jobs_builder.consume(executor, partition_messages)
+            end
+          end
+        end
+
+        jobs.each(&:before_enqueue)
+
+        @scheduler.schedule_consumption(@jobs_queue, jobs)
+      end
+
+      # Waits for all the jobs from a given subscription group to finish before moving forward
+      def wait
+        @jobs_queue.wait(@subscription_group.id)
+      end
+
+      # Waits without blocking the polling
+      #
+      # This should be used only when we no longer plan to use any incoming messages data and we
+      # can safely discard it. We can however use the rebalance information if needed.
+      #
+      # @param wait_until [Proc] until this evaluates to true, we will poll data
+      # @param after_ping [Proc] code that we want to run after each ping (if any)
+      #
+      # @note Performance of this is not relevant (in regards to blocks) because it is used only
+      #   on shutdown and quiet, hence not in the running mode
+      def wait_pinging(wait_until:, after_ping: -> {})
+        until wait_until.call
+          @client.ping
+          after_ping.call
+          sleep(0.2)
+        end
       end
 
-      # @return [Karafka::Connection::Client] wrapped kafka consuming client for a given topic
-      #   consumption
-      def client
-        @client ||= Client.new(@consumer_group)
+      # We can stop client without a problem, as it will reinitialize itself when running the
+      # `#fetch_loop` again. We just need to remember to also reset the runner as it is a long
+      # running one, so with a new connection to Kafka, we need to initialize the state of the
+      # runner and underlying consumers once again.
+      def restart
+        # If there was any problem with processing, before we reset things we need to make sure,
+        # there are no jobs in the queue. Otherwise it could lead to leakage in between client
+        # resetting.
+        @jobs_queue.wait(@subscription_group.id)
+        @jobs_queue.clear(@subscription_group.id)
+        @client.reset
+        @coordinators.reset
+        @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end
   end

data/lib/karafka/connection/listeners_batch.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Abstraction layer around listeners batch.
+    class ListenersBatch
+      include Enumerable
+
+      attr_reader :coordinators
+
+      # @param jobs_queue [JobsQueue]
+      # @return [ListenersBatch]
+      def initialize(jobs_queue)
+        @coordinators = []
+
+        @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
+          consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
+            subscription_groups.size
+          )
+
+          @coordinators << consumer_group_coordinator
+
+          subscription_groups.map do |subscription_group|
+            Connection::Listener.new(
+              consumer_group_coordinator,
+              subscription_group,
+              jobs_queue
+            )
+          end
+        end
+      end
+
+      # Iterates over available listeners and yields each listener
+      # @param block [Proc] block we want to run
+      def each(&block)
+        @batch.each(&block)
+      end
+    end
+  end
+end

data/lib/karafka/connection/messages_buffer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Buffer used to build and store karafka messages built based on raw librdkafka messages.
+    #
+    # Why do we have two buffers? `RawMessagesBuffer` is used to store raw messages and to handle
+    #   cases related to partition revocation and reconnections. It is "internal" to the listening
+    #   process. `MessagesBuffer` on the other hand is used to "translate" those raw messages that
+    #   we know that are ok into Karafka messages and to simplify further work with them.
+    #
+    # While it adds a bit of overhead, it makes conceptual things much easier and it adds only two
+    #   simple hash iterations over messages batch.
+    #
+    # @note This buffer is NOT thread safe. We do not worry about it as we do not use it outside
+    #   of the main listener loop. It can be cleared after the jobs are scheduled with messages
+    #   it stores, because messages arrays are not "cleared" in any way directly and their
+    #   reference stays.
+    class MessagesBuffer
+      attr_reader :size
+
+      # @param subscription_group [Karafka::Routing::SubscriptionGroup]
+      def initialize(subscription_group)
+        @subscription_group = subscription_group
+        @size = 0
+        @groups = Hash.new do |topic_groups, topic|
+          topic_groups[topic] = Hash.new do |partition_groups, partition|
+            partition_groups[partition] = []
+          end
+        end
+      end
+
+      # Remaps raw messages from the raw messages buffer to Karafka messages
+      # @param raw_messages_buffer [RawMessagesBuffer] buffer with raw messages
+      def remap(raw_messages_buffer)
+        clear unless @size.zero?
+
+        # Since it happens "right after" we've received the messages, it is close enough it time
+        # to be used as the moment we received messages.
+        received_at = Time.now
+
+        raw_messages_buffer.each do |topic, partition, messages|
+          @size += messages.count
+
+          ktopic = @subscription_group.topics.find(topic)
+
+          @groups[topic][partition] = messages.map do |message|
+            Messages::Builders::Message.call(
+              message,
+              ktopic,
+              received_at
+            )
+          end
+        end
+      end
+
+      # Allows to iterate over all the topics and partitions messages
+      #
+      # @yieldparam [String] topic name
+      # @yieldparam [Integer] partition number
+      # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
+      def each
+        @groups.each do |topic, partitions|
+          partitions.each do |partition, messages|
+            yield(topic, partition, messages)
+          end
+        end
+      end
+
+      # @return [Boolean] is the buffer empty or does it contain any messages
+      def empty?
+        @size.zero?
+      end
+
+      private
+
+      # Clears the buffer completely
+      def clear
+        @size = 0
+        @groups.clear
+      end
+    end
+  end
+end

data/lib/karafka/connection/pauses_manager.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Partitions pauses management abstraction layer.
+    # It aggregates all the pauses for all the partitions that we're working with.
+    class PausesManager
+      # @return [Karafka::Connection::PausesManager] pauses manager
+      def initialize
+        @pauses = Hash.new do |h, k|
+          h[k] = {}
+        end
+      end
+
+      # Creates or fetches pause tracker of a given topic partition.
+      #
+      # @param topic [::Karafka::Routing::Topic] topic
+      # @param partition [Integer] partition number
+      # @return [Karafka::TimeTrackers::Pause] pause tracker instance
+      def fetch(topic, partition)
+        @pauses[topic][partition] ||= TimeTrackers::Pause.new(
+          timeout: topic.pause_timeout,
+          max_timeout: topic.pause_max_timeout,
+          exponential_backoff: topic.pause_with_exponential_backoff
+        )
+      end
+
+      # Resumes processing of partitions for which pause time has ended.
+      #
+      # @yieldparam [Karafka::Routing::Topic] topic
+      # @yieldparam [Integer] partition number
+      def resume
+        @pauses.each do |topic, partitions|
+          partitions.each do |partition, pause|
+            next unless pause.paused?
+            next unless pause.expired?
+
+            pause.resume
+
+            yield(topic, partition)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/proxy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Usually it is ok to use the `Rdkafka::Consumer` directly because we need 1:1 its
+    # functionality. There are however cases where we want to have extra recoveries or other
+    # handling of errors and settings. This is where this module comes in handy.
+    #
+    # We do not want to wrap and delegate all via a proxy object for performance reasons, but we
+    # do still want to be able to alter some functionalities. This wrapper helps us do it when
+    # it would be needed
+    class Proxy < SimpleDelegator
+      # Errors on which we want to retry
+      RETRYABLE_ERRORS = %i[
+        all_brokers_down
+        timed_out
+      ].freeze
+
+      private_constant :RETRYABLE_ERRORS
+
+      attr_accessor :wrapped
+
+      alias __getobj__ wrapped
+
+      # @param obj [Rdkafka::Consumer, Proxy] rdkafka consumer or consumer wrapped with proxy
+      def initialize(obj)
+        super
+        # Do not allow for wrapping proxy with a proxy. This will prevent a case where we might
+        # wrap an already wrapped object with another proxy level. Simplifies passing consumers
+        # and makes it safe to wrap without type checking
+        @wrapped = obj.is_a?(self.class) ? obj.wrapped : obj
+        @config = ::Karafka::App.config.internal.connection.proxy
+      end
+
+      # Proxies the `#query_watermark_offsets` with extra recovery from timeout problems.
+      # We impose our own custom timeout to make sure, that high-latency clusters and overloaded
+      # clusters can handle our requests.
+      #
+      # @param topic [String] topic name
+      # @param partition [Partition]
+      # @return [Array<Integer, Integer>] watermark offsets
+      def query_watermark_offsets(topic, partition)
+        l_config = @config.query_watermark_offsets
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.query_watermark_offsets(topic, partition, l_config.timeout)
+        end
+      end
+
+      # Similar to `#query_watermark_offsets`, this method can be sensitive to latency. We handle
+      # this the same way
+      #
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList] tpl to get time offsets
+      # @return [Rdkafka::Consumer::TopicPartitionList] tpl with time offsets
+      def offsets_for_times(tpl)
+        l_config = @config.offsets_for_times
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.offsets_for_times(tpl, l_config.timeout)
+        end
+      end
+
+      private
+
+      # Runs expected block of code with few retries on all_brokers_down
+      # librdkafka can return `all_brokers_down` for scenarios when broker is overloaded or not
+      # reachable due to latency.
+      # @param max_attempts [Integer] how many attempts (not retries) should we take before failing
+      #   completely.
+      # @param wait_time [Integer, Float] how many seconds should we wait. It uses `#sleep` of Ruby
+      #   so it needs time in seconds.
+      def with_broker_errors_retry(max_attempts:, wait_time: 1)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise unless RETRYABLE_ERRORS.include?(e.code)
+
+        if attempt <= max_attempts
+          sleep(wait_time)
+
+          retry
+        end
+
+        raise
+      end
+    end
+  end
+end