karafka 2.6.0.beta1 → 2.6.0.rc2

data/lib/karafka/pro/processing/consumer_groups/strategies/dlq/vp.rb

@@ -31,18 +31,15 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Dlq
-            # Dead Letter Queue enabled
-            # Virtual Partitions enabled
+            # - Dead Letter Queue enabled
+            # - Virtual Partitions enabled
             #
-            # In general because we collapse processing in virtual partitions to one on errors, there
-            # is no special action that needs to be taken because we warranty that even with VPs
-            # on errors a retry collapses into a single state and from this single state we can
+            # In general because we collapse processing in virtual partitions to one on errors,
+            # there is no special action that needs to be taken because we warranty that even with
+            # VPson errors a retry collapses into a single state and from this single state we can
             # mark as consumed the message that we are moving to the DLQ.
             module Vp
               # Features for this strategy
@@ -53,6 +50,66 @@ module Karafka
 
               include Strategies::Dlq::Default
               include Strategies::Vp::Default
+
+              # Runs the DLQ strategy and based on it it performs certain operations
+              #
+              # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
+              # In case of `:retry` always `#retry_after_pause` is applied
+              def apply_dlq_flow
+                # Process-critical errors are never dispatched or skipped regardless of the
+                # strategy outcome: the retry pause protects the partition during the critical
+                # shutdown and the failed batch is redelivered after the restart.
+                # We consult `errors_tracker.last` (not the per-consumer consumption cause used
+                # by the OSS strategies, which have no tracker) because it is exactly what the
+                # DLQ strategy callable below receives - this guard judges the same evidence as
+                # the strategy it overrides. The tracker is cleared at attempt zero, so `last`
+                # is always the most recent failure of the current failure streak
+                if critical_error?(errors_tracker.last)
+                  retry_after_pause
+
+                  return
+                end
+
+                # With virtual partitions, a dispatch/skip decision is never made on a
+                # non-collapsed (parallel) run. The deciding consumer operates only on its own
+                # virtual partition subset there: the skippable message would be selected from
+                # an arbitrary subset and the dispatch marking would commit offsets of messages
+                # other virtual partitions never processed. The failure already requested a
+                # collapse, so we retry and let the decision happen on the collapsed, linear
+                # flow where it is deterministic
+                if topic.virtual_partitions? && !collapsed?
+                  retry_after_pause
+
+                  return
+                end
+
+                flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+
+                case flow
+                when :retry
+                  retry_after_pause
+
+                  return
+                when :skip
+                  @_dispatch_to_dlq = false
+                when :dispatch
+                  @_dispatch_to_dlq = true
+                  # Use custom topic if it was returned from the strategy
+                  @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
+                else
+                  raise Karafka::UnsupportedCaseError, flow
+                end
+
+                yield
+
+                # We reset the pause to indicate we will now consider it as "ok".
+                coordinator.pause_tracker.reset
+
+                # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
+                pause(seek_offset, nil, false)
+              ensure
+                @_dispatch_to_dlq_topic = nil
+              end
             end
           end
         end

data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/default.rb

@@ -31,12 +31,8 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
-          # Filtering related init strategies
           module Ftr
             # Only filtering enabled
             module Default

data/lib/karafka/pro/processing/consumer_groups/strategies/ftr/vp.rb

@@ -31,15 +31,12 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           # Filtering related init strategies
           module Ftr
-            # Filtering enabled
-            # VPs enabled
+            # - Filtering enabled
+            # - VPs enabled
             #
             # VPs should operate without any problems with filtering because virtual partitioning
             # happens on the limited set of messages and collective filtering applies the same

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/default.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Lrj

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr.rb

@@ -31,14 +31,11 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Lrj
-            # Filtering enabled
-            # Long-Running Job enabled
+            # - Filtering enabled
+            # - Long-Running Job enabled
             #
             # In general aside from throttling this one will behave the same way as the Lrj
             module Ftr
@@ -69,7 +66,7 @@ module Karafka
 
                       # :seek and :pause are fully handled by handle_post_filtering
                       # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                      return unless coordinator.filter.action == :skip
+                      return unless coordinator.filter.skip?
                     elsif !revoked? && !coordinator.manual_seek?
                       # If not revoked and not throttled, we move to where we were suppose to and
                       # resume

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom.rb

@@ -31,15 +31,12 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Lrj
-            # Long-Running Job enabled
-            # Filtering enabled
-            # Manual offset management enabled
+            # - Long-Running Job enabled
+            # - Filtering enabled
+            # - Manual offset management enabled
             #
             # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
             module FtrMom
@@ -67,7 +64,7 @@ module Karafka
 
                       # :seek and :pause are fully handled by handle_post_filtering
                       # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                      return unless coordinator.filter.action == :skip
+                      return unless coordinator.filter.skip?
                     elsif !revoked? && !coordinator.manual_seek?
                       # If not revoked and not throttled, we move to where we were suppose to and
                       # resume

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_mom_vp.rb

@@ -31,17 +31,13 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
-          # Filtering related init strategies
           module Lrj
-            # Filtering enabled
-            # LRJ enabled
-            # MoM enabled
-            # VPs enabled
+            # - Filtering enabled
+            # - LRJ enabled
+            # - MoM enabled
+            # - VPs enabled
             module FtrMomVp
               # Filtering + LRJ + Mom + VPs
               FEATURES = %i[

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/ftr_vp.rb

@@ -31,15 +31,12 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Lrj
-            # Long-Running Job enabled
-            # Filtering enabled
-            # Virtual Partitions enabled
+            # - Long-Running Job enabled
+            # - Filtering enabled
+            # - Virtual Partitions enabled
             #
             # Behaves same as non-VP because of the aggregated flow in the coordinator.
             module FtrVp

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom.rb

@@ -31,15 +31,12 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           # Namespace for all the LRJ starting strategies
           module Lrj
-            # Long-Running Job enabled
-            # Manual offset management enabled
+            # - Long-Running Job enabled
+            # - Manual offset management enabled
             module Mom
               include Strategies::Default
 

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/mom_vp.rb

@@ -31,16 +31,12 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
-          # Namespace for all the LRJ starting strategies
           module Lrj
-            # Long-Running Job enabled
-            # Manual offset management enabled
-            # Virtual Partitions enabled
+            # - Long-Running Job enabled
+            # - Manual offset management enabled
+            # - Virtual Partitions enabled
             module MomVp
               include Strategies::Mom::Vp
               include Strategies::Lrj::Mom

data/lib/karafka/pro/processing/consumer_groups/strategies/lrj/vp.rb

@@ -31,14 +31,11 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Lrj
-            # Long-Running Job enabled
-            # Virtual Partitions enabled
+            # - Long-Running Job enabled
+            # - Virtual Partitions enabled
             module Vp
               # Same flow as the standard Lrj
               include Strategies::Vp::Default

data/lib/karafka/pro/processing/consumer_groups/strategies/mom/default.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           # Namespace for Mom starting strategies

data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Mom

data/lib/karafka/pro/processing/consumer_groups/strategies/mom/ftr_vp.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Mom

data/lib/karafka/pro/processing/consumer_groups/strategies/mom/vp.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           module Mom

data/lib/karafka/pro/processing/consumer_groups/strategies/vp/default.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module Strategies
           # VP starting strategies
@@ -71,9 +68,13 @@ module Karafka
                     # want to mark all but until the expected location
                     manager.mark_until(message, offset_metadata) if coordinator.finished?
 
-                    return revoked? unless manager.markable?
+                    # When there is no continuous stream of virtually marked offsets from the
+                    # lowest registered one, there is no real offset to commit yet. This is not
+                    # a failure, hence we need to report ownership truthfully: true as long as
+                    # the assignment is still owned
+                    return !revoked? unless manager.markable?
 
-                    manager.markable? ? super(*manager.markable) : revoked?
+                    super(*manager.markable)
                   end
                 end
               ensure
@@ -93,7 +94,11 @@ module Karafka
                   coordinator.synchronize do
                     manager.mark(message, offset_metadata)
                     manager.mark_until(message, offset_metadata) if coordinator.finished?
-                    manager.markable? ? super(*manager.markable) : revoked?
+
+                    # No real offset to commit yet is not a failure, see `#mark_as_consumed`
+                    return !revoked? unless manager.markable?
+
+                    super(*manager.markable)
                   end
                 end
               ensure

data/lib/karafka/pro/processing/consumer_groups/strategy_selector.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         # Selector of appropriate processing strategy matching topic combinations
         # When using Karafka Pro, there is a different set of strategies than for regular, as

data/lib/karafka/pro/processing/consumer_groups/subscription_groups_coordinator.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         # Uses the jobs queue API to lock (pause) and unlock (resume) operations of a given
         # subscription group. It is abstracted away from jobs queue on this layer because we do

data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/balanced.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module VirtualPartitions
           module Distributors

data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/base.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         # Processing components for virtual partitions
         module VirtualPartitions
@@ -41,7 +38,8 @@ module Karafka
           module Distributors
             # Base class for all virtual partition distributors
             class Base
-              # @param config [Karafka::Pro::Routing::Features::ConsumerGroups::VirtualPartitions::Config]
+              # @param config
+              #   [Karafka::Pro::Routing::Features::ConsumerGroups::VirtualPartitions::Config]
               def initialize(config)
                 @config = config
               end

data/lib/karafka/pro/processing/consumer_groups/virtual_partitions/distributors/consistent.rb

@@ -31,9 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
-      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
-      # lands.
       module ConsumerGroups
         module VirtualPartitions
           module Distributors

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -151,7 +151,10 @@ module Karafka
             @statistics[:waiting] -= @in_waiting[group_id].size
             @in_waiting[group_id].clear
             @locks[group_id].clear
-            @async_locking = false
+            # Recompute the async-locking fast-path flag from the remaining groups rather than
+            # forcing it off: this clear is per subscription group (run on that group's recovery)
+            # and must not disable async locks still held by other groups
+            @async_locking = @locks.any? { |_group_id, locks| locks.any? }
 
             # We unlock it just in case it was blocked when clearing started
             tick(group_id)

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -92,7 +92,8 @@ module Karafka
 
           # Runs the shutdown jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for scheduling
+          # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Shutdown>] jobs for
+          #   scheduling
           def on_schedule_shutdown(jobs_array)
             @mutex.synchronize do
               schedule_shutdown(jobs_array)
@@ -101,7 +102,8 @@ module Karafka
 
           # Runs the idle jobs scheduling flow under a mutex
           #
-          # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for scheduling
+          # @param jobs_array [Array<Karafka::Processing::ConsumerGroups::Jobs::Idle>] jobs for
+          #   scheduling
           def on_schedule_idle(jobs_array)
             @mutex.synchronize do
               schedule_idle(jobs_array)

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -31,7 +31,6 @@
 module Karafka
   module Pro
     module Processing
-      # Namespace for Pro schedulers
       module Schedulers
         # Optimizes scheduler that takes into consideration of execution time needed to process
         # messages from given topics partitions. It uses the non-preemptive LJF algorithm
@@ -109,7 +108,8 @@ module Karafka
               messages = job.messages
               message = messages.first
 
-              perf_tracker.processing_time_p95(message.topic, message.partition) * messages.size
+              perf_tracker.processing_time_p95(job.group_id, message.topic, message.partition) *
+                messages.size
             else
               # LJF will set first the most expensive, but we want to run the zero cost jobs
               # related to the lifecycle always first. That is why we "emulate" that they

data/lib/karafka/pro/recurring_tasks/consumer.rb

@@ -71,9 +71,16 @@ module Karafka
                 message.offset - 1
               )
             when "command"
-              @executor.apply_command(payload)
+              # During replaying commands are only buffered. Applying them immediately would be
+              # pointless as the end-of-replay synchronization with the last stored schedule
+              # state would overwrite their effects. They are applied after the synchronization
+              if @executor.replaying?
+                @executor.replay_command(payload)
+
+                next
+              end
 
-              next if @executor.replaying?
+              @executor.apply_command(payload)
 
               # Execute on each incoming command to have nice latency but only after replaying
               # During replaying we should not execute because there may be more state changes

data/lib/karafka/pro/recurring_tasks/contracts/config.rb

@@ -31,7 +31,6 @@
 module Karafka
   module Pro
     module RecurringTasks
-      # Recurring Tasks related contracts
       module Contracts
         # Makes sure, all the expected config is defined as it should be
         class Config < Karafka::Contracts::Base

data/lib/karafka/pro/recurring_tasks/executor.rb

@@ -76,6 +76,14 @@ module Karafka
           end
         end
 
+        # Buffers a command received during replay, so it can be applied once the in-memory
+        # state is synchronized with the last stored schedule state. Commands applied directly
+        # during replay would have their effects overwritten by this synchronization
+        # @param command_hash [Hash] deserialized command data
+        def replay_command(command_hash)
+          @catchup_commands << command_hash
+        end
+
         # Updates the catchup state
         # @param schedule_hash [Hash] deserialized schedule hash hash
         def update_state(schedule_hash)
@@ -100,24 +108,28 @@ module Karafka
             return
           end
 
-          # If the schedule version we have in Kafka is higher than ours, we cannot proceed
-          # This prevents us from applying older changes to a new schedule
-          if @catchup_schedule[:schedule_version] > schedule.version
-            @incompatible = true
+          # Commands may have been received without any schedule state stored in Kafka. In such
+          # cases there is nothing to synchronize and only the commands are applied
+          if @catchup_schedule
+            # If the schedule version we have in Kafka is higher than ours, we cannot proceed
+            # This prevents us from applying older changes to a new schedule
+            if Gem::Version.new(@catchup_schedule[:schedule_version]) > Gem::Version.new(schedule.version)
+              @incompatible = true
 
-            return
-          end
+              return
+            end
 
-          # Now we can synchronize the in-memory state based on the last state stored in Kafka
-          schedule.each do |task|
-            stored_task = @catchup_schedule[:tasks][task.id.to_sym]
+            # Now we can synchronize the in-memory state based on the last state stored in Kafka
+            schedule.each do |task|
+              stored_task = @catchup_schedule[:tasks][task.id.to_sym]
 
-            next unless stored_task
+              next unless stored_task
 
-            stored_previous_time = stored_task[:previous_time]
-            task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
+              stored_previous_time = stored_task[:previous_time]
+              task.previous_time = stored_previous_time.zero? ? 0 : Time.at(stored_previous_time)
 
-            stored_task[:enabled] ? task.enable : task.disable
+              stored_task[:enabled] ? task.enable : task.disable
+            end
           end
 
           @catchup_commands.each do |cmd|

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb

@@ -33,8 +33,6 @@ module Karafka
     module Routing
       module Features
         module ConsumerGroups
-          # Alterations to the direct assignments that allow us to do stable direct assignments
-          # without working with consumer groups dynamic assignments
           class DirectAssignments < Base
             # Extension allowing us to select correct subscriptions and assignments based on the
             # expanded routing setup

data/lib/karafka/pro/routing/features/consumer_groups/long_running_job.rb

@@ -30,9 +30,7 @@
 
 module Karafka
   module Pro
-    # Namespace for Pro routing enhancements
     module Routing
-      # Namespace for additional Pro features
       module Features
         module ConsumerGroups
           # Long-Running Jobs feature config and DSL namespace.

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/proxy.rb

@@ -36,8 +36,8 @@ module Karafka
           class Multiplexing < Base
             # Allows for multiplexing setup inside a consumer group definition
             module Proxy
-              # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
-              #   disabling dynamic multiplexing
+              # @param min [Integer, nil] min multiplexing count or nil to set it to max,
+              #   effectively disabling dynamic multiplexing
               # @param max [Integer] max multiplexing count
               # @param boot [Integer] how many listeners should we start during boot by default
               # @param scale_delay [Integer] number of ms of delay before applying any scale

data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job.rb

@@ -30,9 +30,7 @@
 
 module Karafka
   module Pro
-    # Namespace for Pro routing enhancements
     module Routing
-      # Namespace for additional Pro features
       module Features
         module ConsumerGroups
           # Non Blocking Job is just an alias for LRJ.

data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb

@@ -33,12 +33,6 @@ module Karafka
     module Routing
       module Features
         module ConsumerGroups
-          # This feature allows for saving and retrieving offset metadata with custom deserialization
-          # support. It allows for storing extra data during commits that can be then used to alter
-          # the processing flow after a rebalance.
-          #
-          # @note Because this feature has zero performance impact and makes no queries to Kafka
-          #   unless requested, it is always enabled.
           class OffsetMetadata < Base
             # Empty string not to create it on each deserialization
             EMPTY_STRING = ""