karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/consumer_groups/strategies/dlq.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # When using dead letter queue, processing won't stop after defined number of retries
+        # upon encountering non-critical errors but the messages that error will be moved to a
+        # separate topic with their payload and metadata, so they can be handled differently.
+        module Dlq
+          include Default
+
+          # Apply strategy when only dead letter queue is turned on
+          FEATURES = %i[
+            dead_letter_queue
+          ].freeze
+
+          # Override of the standard `#mark_as_consumed` in order to handle the pause tracker
+          # reset in case DLQ is marked as fully independent. When DLQ is marked independent,
+          # any offset marking causes the pause count tracker to reset. This is useful when
+          # the error is not due to the collective batch operations state but due to intermediate
+          # "crawling" errors that move with it
+          #
+          # @see `Strategies::Default#mark_as_consumed` for more details
+          # @param message [Messages::Message]
+          def mark_as_consumed(message)
+            # If we are not retrying pause count is already 0, no need to try to reset the state
+            return super unless retrying?
+            # If we do not use independent marking on DLQ, we just mark as consumed
+            return super unless topic.dead_letter_queue.independent?
+            # If we were not able to mark no need to reset
+            return false unless super
+
+            coordinator.pause_tracker.reset
+
+            true
+          end
+
+          # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
+          # DLQ was configured with the `independent` flag.
+          #
+          # @see `Strategies::Default#mark_as_consumed!` for more details
+          # @param message [Messages::Message]
+          def mark_as_consumed!(message)
+            return super unless retrying?
+            return super unless topic.dead_letter_queue.independent?
+            return false unless super
+
+            coordinator.pause_tracker.reset
+
+            true
+          end
+
+          # When manual offset management is on, we do not mark anything as consumed automatically
+          # and we rely on the user to figure things out
+          def handle_after_consume
+            return if revoked?
+
+            if coordinator.success?
+              coordinator.pause_tracker.reset
+
+              return if coordinator.manual_pause?
+
+              mark_as_consumed(messages.last)
+            elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+              retry_after_pause
+            # If we've reached number of retries that we could, we need to skip the first message
+            # that was not marked as consumed, pause and continue, while also moving this message
+            # to the dead topic
+            else
+              # We reset the pause to indicate we will now consider it as "ok".
+              coordinator.pause_tracker.reset
+
+              skippable_message, = find_skippable_message
+
+              # Send skippable message to the dql topic
+              dispatch_to_dlq(skippable_message)
+
+              # We mark the broken message as consumed and move on
+              if mark_after_dispatch?
+                mark_dispatched_to_dlq(skippable_message)
+
+                return if revoked?
+              else
+                self.seek_offset = skippable_message.offset + 1
+              end
+
+              # We pause to backoff once just in case.
+              pause(seek_offset, nil, false)
+            end
+          end
+
+          # Finds the message may want to skip (all, starting from first)
+          # @private
+          # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+          #   information if this message was from marked offset or figured out via mom flow
+          def find_skippable_message
+            skippable_message = messages.raw.find do |msg|
+              coordinator.marked? && msg.offset == seek_offset
+            end
+
+            # If we don't have the message matching the last comitted offset, it means that
+            # user operates with manual offsets and we're beyond the batch in which things
+            # broke for the first time. Then we skip the first (as no markings) and we
+            # move on one by one.
+            skippable_message ? [skippable_message, true] : [messages.first, false]
+          end
+
+          # Moves the broken message into a separate queue defined via the settings
+          # @private
+          # @param skippable_message [Karafka::Messages::Message] message we are skipping that also
+          #   should go to the dlq topic
+          def dispatch_to_dlq(skippable_message)
+            producer.public_send(
+              topic.dead_letter_queue.dispatch_method,
+              topic: topic.dead_letter_queue.topic,
+              payload: skippable_message.raw_payload
+            )
+
+            # Notify about dispatch on the events bus
+            monitor.instrument(
+              "dead_letter_queue.dispatched",
+              caller: self,
+              message: skippable_message
+            )
+          end
+
+          # @return [Boolean] should we mark given message as consumed after dispatch. For default
+          #  non MOM strategies if user did not explicitly tell us not to, we mark it. Default is
+          #  `nil`, which means `true` in this case. If user provided alternative value, we go
+          #  with it.
+          def mark_after_dispatch?
+            return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+            topic.dead_letter_queue.mark_after_dispatch
+          end
+
+          # Marks message that went to DLQ (if applicable) based on the requested method
+          # @param skippable_message [Karafka::Messages::Message]
+          def mark_dispatched_to_dlq(skippable_message)
+            case topic.dead_letter_queue.marking_method
+            when :mark_as_consumed
+              mark_as_consumed(skippable_message)
+            when :mark_as_consumed!
+              mark_as_consumed!(skippable_message)
+            else
+              # This should never happen. Bug if encountered. Please report
+              raise Karafka::Errors::UnsupportedCaseError
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # Same as pure dead letter queue but we do not marked failed message as consumed
+        module DlqMom
+          include Dlq
+
+          # Apply strategy when dlq is on with manual offset management
+          FEATURES = %i[
+            dead_letter_queue
+            manual_offset_management
+          ].freeze
+
+          # When manual offset management is on, we do not mark anything as consumed automatically
+          # and we rely on the user to figure things out
+          def handle_after_consume
+            return if revoked?
+
+            if coordinator.success?
+              coordinator.pause_tracker.reset
+            elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+              retry_after_pause
+            # If we've reached number of retries that we could, we need to skip the first message
+            # that was not marked as consumed, pause and continue, while also moving this message
+            # to the dead topic
+            else
+              # We reset the pause to indicate we will now consider it as "ok".
+              coordinator.pause_tracker.reset
+
+              skippable_message, = find_skippable_message
+
+              dispatch_to_dlq(skippable_message)
+
+              # We mark the broken message as consumed and move on
+              if mark_after_dispatch?
+                mark_dispatched_to_dlq(skippable_message)
+
+                return if revoked?
+              else
+                # Save the next offset we want to go with after moving given message to DLQ
+                # Without this, we would not be able to move forward and we would end up
+                # in an infinite loop trying to un-pause from the message we've already processed
+                # Of course, since it's a MoM a rebalance or kill, will move it back as no
+                # offsets are being committed
+                self.seek_offset = skippable_message.offset + 1
+              end
+
+              pause(seek_offset, nil, false)
+            end
+          end
+
+          # @return [Boolean] should we mark given message as consumed after dispatch. For
+          #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default is
+          #  `nil`, which means `false` in this case. If user provided alternative value, we go
+          #  with it.
+          #
+          # @note Please note, this is the opposite behavior than in case of AOM strategies.
+          def mark_after_dispatch?
+            return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+            topic.dead_letter_queue.mark_after_dispatch
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategies/mom.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module Strategies
+        # When using manual offset management, we do not mark as consumed after successful processing
+        module Mom
+          include Default
+
+          # Apply strategy when only manual offset management is turned on
+          FEATURES = %i[
+            manual_offset_management
+          ].freeze
+
+          # When manual offset management is on, we do not mark anything as consumed automatically
+          # and we rely on the user to figure things out
+          def handle_after_consume
+            return if revoked?
+
+            if coordinator.success?
+              coordinator.pause_tracker.reset
+            else
+              retry_after_pause
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/strategy_selector.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Selector of appropriate processing strategy matching topic combinations
+      class StrategySelector
+        attr_reader :strategies
+
+        # Features we support in the OSS offering.
+        SUPPORTED_FEATURES = %i[
+          active_job
+          manual_offset_management
+          dead_letter_queue
+        ].freeze
+
+        # Initializes the strategy selector and preloads all strategies
+        def initialize
+          # We load them once for performance reasons not to do too many lookups
+          @strategies = find_all
+        end
+
+        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+        #   strategy
+        # @return [Module] module with proper strategy
+        def find(topic)
+          feature_set = SUPPORTED_FEATURES.map do |feature_name|
+            topic.public_send("#{feature_name}?") ? feature_name : nil
+          end
+
+          feature_set.compact!
+
+          @strategies.find do |strategy|
+            strategy::FEATURES.sort == feature_set.sort
+          end || raise(Errors::StrategyNotFoundError, topic.name)
+        end
+
+        private
+
+        # @return [Array<Module>] available strategies
+        def find_all
+          Strategies
+            .constants
+            .delete_if { |k| k == :Base }
+            .map { |k| Strategies.const_get(k) }
+            .uniq
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinator.rb

@@ -1,217 +1,10 @@
 # frozen_string_literal: true
 
+# Backwards-compatible alias kept for karafka-web gem that references the old, un-namespaced
+# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
-    # Basic coordinator that allows us to provide coordination objects into consumers.
-    #
-    # This is a wrapping layer to simplify management of work to be handled around consumption.
-    #
-    # @note This coordinator needs to be thread safe. Some operations are performed only in the
-    #   listener thread, but we go with thread-safe by default for all not to worry about potential
-    #   future mistakes.
-    class Coordinator
-      extend Forwardable
-      include Core::Helpers::Time
-
-      attr_reader :pause_tracker, :seek_offset, :topic, :partition
-
-      # This can be set directly on the listener because it can be triggered on first run without
-      # any messages
-      attr_accessor :eofed
-
-      # Last polled at time set based on the incoming last poll time
-      attr_accessor :last_polled_at
-
-      def_delegators :@pause_tracker, :attempt, :paused?
-
-      # @param topic [Karafka::Routing::Topic]
-      # @param partition [Integer]
-      # @param pause_tracker [Karafka::TimeTrackers::Pause] pause tracker for given topic partition
-      def initialize(topic, partition, pause_tracker)
-        @topic = topic
-        @partition = partition
-        @pause_tracker = pause_tracker
-        @revoked = false
-        @consumptions = {}
-        @running_jobs = Hash.new { |h, k| h[k] = 0 }
-        @manual_pause = false
-        @manual_seek = false
-        @mutex = Mutex.new
-        @marked = false
-        @failure = false
-        @eofed = false
-        @changed_at = monotonic_now
-        @last_polled_at = @changed_at
-      end
-
-      # Starts the coordinator for given consumption jobs
-      # @param messages [Array<Karafka::Messages::Message>] batch of message for which we are
-      #   going to coordinate work. Not used with regular coordinator.
-      def start(messages)
-        @failure = false
-        @running_jobs[:consume] = 0
-        # We need to clear the consumption results hash here, otherwise we could end up storing
-        # consumption results of consumer instances we no longer control
-        @consumptions.clear
-
-        # When starting to run, no pause is expected and no manual pause as well
-        @manual_pause = false
-
-        # No user invoked seeks on a new run
-        @manual_seek = false
-
-        # We set it on the first encounter and never again, because then the offset setting
-        # should be up to the consumers logic (our or the end user)
-        # Seek offset needs to be always initialized as for case where manual offset management
-        # is turned on, we need to have reference to the first offset even in case of running
-        # multiple batches without marking any messages as consumed. Rollback needs to happen to
-        # the last place we know of or the last message + 1 that was marked
-        #
-        # It is however worth keeping in mind, that this may need to be used with `#marked?` to
-        # make sure that the first offset is an offset that has been marked.
-        @seek_offset ||= messages.first.offset
-      end
-
-      # @param offset [Integer] message offset
-      def seek_offset=(offset)
-        synchronize do
-          @marked = true
-          @seek_offset = offset
-        end
-      end
-
-      # Increases number of jobs that we handle with this coordinator
-      # @param job_type [Symbol] type of job that we want to increment
-      def increment(job_type)
-        synchronize do
-          @running_jobs[job_type] += 1
-          @changed_at = monotonic_now
-        end
-      end
-
-      # Decrements number of jobs we handle at the moment
-      # @param job_type [Symbol] type of job that we want to decrement
-      def decrement(job_type)
-        synchronize do
-          @running_jobs[job_type] -= 1
-          @changed_at = monotonic_now
-
-          return @running_jobs[job_type] unless @running_jobs[job_type].negative?
-
-          # This should never happen. If it does, something is heavily out of sync. Please reach
-          # out to us if you encounter this
-          raise Karafka::Errors::InvalidCoordinatorStateError, "Was zero before decrementation"
-        end
-      end
-
-      # Is all the consumption done and finished successfully for this coordinator
-      # We do not say we're successful until all work is done, because running work may still
-      # crash.
-      # @note This is only used for consume synchronization
-      def success?
-        synchronize do
-          @running_jobs[:consume].zero? && @consumptions.values.all?(&:success?)
-        end
-      end
-
-      # Mark given consumption on consumer as successful
-      # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
-      def success!(consumer)
-        synchronize do
-          consumption(consumer).success!
-        end
-      end
-
-      # Mark given consumption on consumer as failed
-      # @param consumer [Karafka::BaseConsumer] consumer that failed
-      # @param error [StandardError] error that occurred
-      def failure!(consumer, error)
-        synchronize do
-          @failure = true
-          consumption(consumer).failure!(error)
-        end
-      end
-
-      # @return [Boolean] true if any of work we were running failed
-      def failure?
-        @failure
-      end
-
-      # Marks given coordinator for processing group as revoked
-      #
-      # This is invoked in two places:
-      #   - from the main listener loop when we detect revoked partitions
-      #   - from the consumer in case checkpointing fails
-      #
-      # This means, we can end up having consumer being aware that it was revoked prior to the
-      # listener loop dispatching the revocation job. It is ok, as effectively nothing will be
-      # processed until revocation jobs are done.
-      def revoke
-        synchronize { @revoked = true }
-      end
-
-      # @return [Boolean] is the partition we are processing revoked or not
-      def revoked?
-        @revoked
-      end
-
-      # @return [Boolean] did we reach end of partition when polling data
-      def eofed?
-        @eofed
-      end
-
-      # @return [Boolean] was the new seek offset assigned at least once. This is needed because
-      #   by default we assign seek offset of a first message ever, however this is insufficient
-      #   for DLQ in a scenario where the first message would be broken. We would never move
-      #   out of it and would end up in an endless loop.
-      def marked?
-        @marked
-      end
-
-      # Store in the coordinator info, that this pause was done manually by the end user and not
-      # by the system itself
-      def manual_pause
-        @manual_pause = true
-      end
-
-      # @return [Boolean] are we in a pause that was initiated by the user
-      def manual_pause?
-        paused? && @manual_pause
-      end
-
-      # Marks seek as manual for coordination purposes
-      def manual_seek
-        @manual_seek = true
-      end
-
-      # @return [Boolean] did a user invoke seek in the current operations scope
-      def manual_seek?
-        @manual_seek
-      end
-
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
-        @consumptions[consumer] ||= Processing::Result.new
-      end
-
-      # Allows to run synchronized (locked) code that can operate only from a given thread
-      #
-      # @note We check if mutex is not owned already by the current thread so we won't end up with
-      #   a deadlock in case user runs coordinated code from inside of his own lock
-      #
-      # @note This is internal and should **not** be used to synchronize user-facing code.
-      #   Otherwise user indirectly could cause deadlocks or prolonged locks by running his logic.
-      #   This can and should however be used for multi-thread strategy applications and other
-      #   internal operations locks.
-      def synchronize(&)
-        if @mutex.owned?
-          yield
-        else
-          @mutex.synchronize(&)
-        end
-      end
-    end
+    # @see ConsumerGroups::Coordinator
+    Coordinator = ConsumerGroups::Coordinator
   end
 end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -1,65 +1,10 @@
 # frozen_string_literal: true
 
+# Backwards-compatible alias kept for karafka-testing gem that references the old, un-namespaced
+# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
-    # Coordinators builder used to build coordinators per topic partition
-    #
-    # It provides direct pauses access for revocation
-    #
-    # @note This buffer operates only from the listener loop, thus we do not have to make it
-    #   thread-safe.
-    class CoordinatorsBuffer
-      include Helpers::ConfigImporter.new(
-        coordinator_class: %i[internal processing coordinator_class]
-      )
-
-      # @param topics [Karafka::Routing::Topics]
-      def initialize(topics)
-        @pauses_manager = Connection::PausesManager.new
-        @coordinators = Hash.new { |h, k| h[k] = {} }
-        @topics = topics
-      end
-
-      # @param topic_name [String] topic name
-      # @param partition [Integer] partition number
-      # @return [Karafka::Processing::Coordinator] found or created coordinator
-      def find_or_create(topic_name, partition)
-        @coordinators[topic_name][partition] ||= begin
-          routing_topic = @topics.find(topic_name)
-
-          coordinator_class.new(
-            routing_topic,
-            partition,
-            @pauses_manager.fetch(routing_topic, partition)
-          )
-        end
-      end
-
-      # Resumes processing of partitions for which pause time has ended.
-      # @yieldparam [String] topic name
-      # @yieldparam [Integer] partition number
-      def resume(&)
-        @pauses_manager.resume(&)
-      end
-
-      # @param topic_name [String] topic name
-      # @param partition [Integer] partition number
-      def revoke(topic_name, partition)
-        return unless @coordinators[topic_name].key?(partition)
-
-        # The fact that we delete here does not change the fact that the executor still holds the
-        # reference to this coordinator. We delete it here, as we will no longer process any
-        # new stuff with it and we may need a new coordinator if we regain this partition, but the
-        # coordinator may still be in use
-        @coordinators[topic_name].delete(partition).revoke
-      end
-
-      # Clears coordinators and re-created the pauses manager
-      # This should be used only for critical errors recovery
-      def reset
-        @pauses_manager = Connection::PausesManager.new
-        @coordinators.clear
-      end
-    end
+    # @see ConsumerGroups::CoordinatorsBuffer
+    CoordinatorsBuffer = ConsumerGroups::CoordinatorsBuffer
   end
 end

data/lib/karafka/processing/jobs_queue.rb

@@ -14,10 +14,13 @@ module Karafka
     #   we use a single workers poll that can have granular scheduling.
     class JobsQueue
       include Helpers::ConfigImporter.new(
-        concurrency: %i[concurrency],
         tick_interval: %i[internal tick_interval]
       )
 
+      # Set via writer because of a circular dependency: the queue must exist before the pool
+      # (workers need the queue at construction), but the queue needs the pool for concurrency.
+      attr_writer :pool
+
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
@@ -56,10 +59,15 @@ module Karafka
       #
       # @param job [Jobs::Base] job that we want to run
       def <<(job)
-        # We do not push the job if the queue is closed as it means that it would anyhow not be
-        # executed
         return if @queue.closed?
 
+        # nil is used by WorkersPool to signal a worker to exit during downscaling.
+        # Passed straight through to the raw queue, bypassing statistics and tracking.
+        unless job
+          @queue << job
+          return
+        end
+
         @mutex.synchronize do
           group = @in_processing[job.group_id]
 
@@ -70,7 +78,7 @@ module Karafka
           # Assume that moving to queue means being picked up immediately not to create stats
           # race conditions because of pop overhead. If there are workers available, we assume
           # work is going to be handled as we never reject enqueued jobs
-          if @statistics[:busy] < concurrency
+          if @statistics[:busy] < @pool.size
             @statistics[:busy] += 1
           else
             # If system is fully loaded, it means this job is indeed enqueued

data/lib/karafka/processing/partitioner.rb

@@ -1,24 +1,10 @@
 # frozen_string_literal: true
 
+# Backwards-compatible alias kept for external code that references the old, un-namespaced
+# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
-    # Basic partitioner for work division
-    # It does not divide any work.
-    class Partitioner
-      # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
-      def initialize(subscription_group)
-        @subscription_group = subscription_group
-      end
-
-      # @param _topic [String] topic name
-      # @param messages [Array<Karafka::Messages::Message>] karafka messages
-      # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
-      #   be used with those messages
-      # @yieldparam [Integer] group id
-      # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-      def call(_topic, messages, _coordinator)
-        yield(0, messages)
-      end
-    end
+    # @see ConsumerGroups::Partitioner
+    Partitioner = ConsumerGroups::Partitioner
   end
 end

data/lib/karafka/processing/schedulers/default.rb

@@ -13,7 +13,8 @@ module Karafka
 
         # Schedules jobs in the fifo order
         #
-        # @param jobs_array [Array<Karafka::Processing::Jobs::Consume>] jobs we want to schedule
+        # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Consume>] jobs we
+        #   want to schedule
         def on_schedule_consumption(jobs_array)
           jobs_array.each do |job|
             @queue << job