karafka 2.4.18 → 2.5.0.beta1

data/lib/karafka/pro/cli/parallel_segments.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    # Pro related CLI commands
+    module Cli
+      # CLI entry-point for parallel segments management commands
+      class ParallelSegments < Karafka::Cli::Base
+        include Helpers::Colorize
+        include Helpers::ConfigImporter.new(
+          kafka_config: %i[kafka]
+        )
+
+        desc 'Allows for parallel segments management'
+
+        option(
+          :groups,
+          'Names of consumer groups on which we want to run the command. All if not provided',
+          Array,
+          %w[
+            --groups
+            --consumer_groups
+          ]
+        )
+
+        # Some operations may not be allowed to run again after data is set in certain ways.
+        # For example if a distribution command is invoked when the parallel group segment
+        # consumer groups already have offsets set, we will fail unless user wants to force it.
+        # This prevents users from accidentally running the command in such ways that would cause
+        # their existing distributed offsets to be reset.
+        option(
+          :force,
+          'Should an operation on the parallel segments consumer group be forced',
+          TrueClass,
+          %w[
+            --force
+          ]
+        )
+
+        # @param action [String] action we want to take
+        def call(action = 'distribute')
+          case action
+          when 'distribute'
+            Distribute.new(options).call
+          when 'collapse'
+            Collapse.new(options).call
+          when 'reset'
+            Collapse.new(options).call
+            Distribute.new(options).call
+          else
+            raise ::ArgumentError, "Invalid topics action: #{action}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/connection/manager.rb

@@ -19,14 +19,9 @@ module Karafka
       class Manager < Karafka::Connection::Manager
         include Core::Helpers::Time
 
-        # How long should we wait after a rebalance before doing anything on a consumer group
-        #
-        # @param scale_delay [Integer] How long should we wait before making any changes. Any
-        #   change related to this consumer group will postpone the scaling operations. This is
-        #   done that way to prevent too many friction in the cluster. It is 1 minute by default
-        def initialize(scale_delay = 60 * 1_000)
+        # Creates new manager instance
+        def initialize
           super()
-          @scale_delay = scale_delay
           @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
@@ -201,7 +196,7 @@ module Karafka
 
             next unless multiplexing.active?
             next unless multiplexing.dynamic?
-            # If we cannot downscale, do not
+            # If we cannot upscale, do not
             next if sg_listeners.count(&:active?) >= multiplexing.max
 
             sg_listeners.each do |sg_listener|
@@ -234,6 +229,8 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
+          @scale_delay ||= sg_listeners.first.subscription_group.multiplexing.scale_delay
+
           sg_listeners.all? do |sg_listener|
             # If a listener is not active, we do not take it into consideration when looking at
             # the stability data

data/lib/karafka/pro/encryption.rb

@@ -33,6 +33,14 @@ module Karafka
           # Encryption for WaterDrop
           config.producer.middleware.append(Messages::Middleware.new)
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -50,7 +50,7 @@ module Karafka
           partition = messages.metadata.partition
 
           samples = @processing_times[topic][partition]
-          samples << event[:time] / messages.count
+          samples << event[:time] / messages.size
 
           return unless samples.size > SAMPLES_COUNT
 

data/lib/karafka/pro/iterator/expander.rb

@@ -21,8 +21,10 @@ module Karafka
       # - { 'topic1' => 100 } - means we run all partitions from the offset 100
       # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
       # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
-      # - { 'topic1' => { 1 => true } } - will pick first offset not consumed on this CG for p 1
-      # - { 'topic1' => true } - will pick first offset not consumed on this CG for all p
+      # - { 'topic1' => { 1 => true } } - will pick first offset on this CG for partition 1
+      # - { 'topic1' => true } - will pick first offset for all partitions
+      # - { 'topic1' => :earliest } - will pick earliest offset for all partitions
+      # - { 'topic1' => :latest } - will pick latest (high-watermark) for all partitions
       class Expander
         # Expands topics to which we want to subscribe with partitions information in case this
         # info is not provided.
@@ -80,7 +82,7 @@ module Karafka
             .find { |topic| topic.fetch(:topic_name) == name }
             .tap { |topic| topic || raise(Errors::TopicNotFoundError, name) }
             .fetch(:partitions)
-            .count
+            .size
         end
       end
     end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -14,6 +14,11 @@ module Karafka
       # This builder resolves that and builds a tpl to which we can safely subscribe the way
       # we want it.
       class TplBuilder
+        # Supported named offset positions that we can reference via their name
+        SUPPORTED_NAMED_POSITIONS = %w[earliest latest].freeze
+
+        private_constant :SUPPORTED_NAMED_POSITIONS
+
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
@@ -28,6 +33,7 @@ module Karafka
           resolve_partitions_with_exact_offsets
           resolve_partitions_with_negative_offsets
           resolve_partitions_with_time_offsets
+          resolve_partitions_with_named_offsets
           resolve_partitions_with_cg_expectations
 
           # Final tpl with all the data
@@ -143,6 +149,23 @@ module Karafka
           end
         end
 
+        # If we get named offsets, we can just remap them to librdkafka special offset positions
+        def resolve_partitions_with_named_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Skip offsets that do not match our named expectations
+              named_offset = offset.to_s
+
+              next unless SUPPORTED_NAMED_POSITIONS.include?(named_offset)
+
+              @mapped_topics[name][partition] = -1 if named_offset == 'latest'
+              @mapped_topics[name][partition] = -2 if named_offset == 'earliest'
+            end
+          end
+        end
+
         # Fetches last used offsets for those partitions for which we want to consume from last
         # moment where given consumer group has finished
         # This is indicated by given partition value being set to `true`.

data/lib/karafka/pro/loader.rb

@@ -60,6 +60,15 @@ module Karafka
           Processing::SubscriptionGroupsCoordinator.instance
         end
 
+        # Runs operations needed after fork in swarm for features that need it
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer] pre fork producer instance that may be
+        #   needed to be replaced with newly changed one post-fork.
+        def post_fork(config, pre_fork_producer)
+          features.each { |feature| feature.post_fork(config, pre_fork_producer) }
+        end
+
         private
 
         # @return [Array<Module>] extra non-routing related pro features and routing components
@@ -84,6 +93,7 @@ module Karafka
           icfg.connection.manager = Connection::Manager.new
 
           icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.errors_tracker_class = Processing::Coordinators::ErrorsTracker
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue

data/lib/karafka/pro/processing/coordinator.rb

@@ -10,6 +10,9 @@ module Karafka
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
         extend Forwardable
+        include Helpers::ConfigImporter.new(
+          errors_tracker_class: %i[internal processing errors_tracker_class]
+        )
 
         def_delegators :@collapser, :collapsed?, :collapse_until!
 
@@ -20,7 +23,7 @@ module Karafka
           super
 
           @executed = []
-          @errors_tracker = Coordinators::ErrorsTracker.new
+          @errors_tracker = errors_tracker_class.new(topic, partition)
           @flow_mutex = Mutex.new
           # Lock for user code synchronization
           # We do not want to mix coordinator lock with the user lock not to create cases where

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -13,25 +13,47 @@ module Karafka
         class ErrorsTracker
           include Enumerable
 
+          # @return [Karafka::Routing::Topic] topic of this error tracker
+          attr_reader :topic
+
+          # @return [Integer] partition of this error tracker
+          attr_reader :partition
+
+          # @return [Hash]
+          attr_reader :counts
+
           # Max errors we keep in memory.
           # We do not want to keep more because for DLQ-less this would cause memory-leaks.
+          # We do however count per class for granular error counting
           STORAGE_LIMIT = 100
 
           private_constant :STORAGE_LIMIT
 
-          def initialize
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer]
+          # @param limit [Integer] max number of errors we want to keep for reference when
+          #   implementing custom error handling.
+          # @note `limit` does not apply to the counts. They will work beyond the number of errors
+          #   occurring
+          def initialize(topic, partition, limit: STORAGE_LIMIT)
             @errors = []
+            @counts = Hash.new { |hash, key| hash[key] = 0 }
+            @topic = topic
+            @partition = partition
+            @limit = limit
           end
 
           # Clears all the errors
           def clear
             @errors.clear
+            @counts.clear
           end
 
           # @param error [StandardError] adds the error to the tracker
           def <<(error)
-            @errors.shift if @errors.size >= STORAGE_LIMIT
+            @errors.shift if @errors.size >= @limit
             @errors << error
+            @counts[error.class] += 1
           end
 
           # @return [Boolean] is the error tracker empty
@@ -41,7 +63,9 @@ module Karafka
 
           # @return [Integer] number of elements
           def size
-            count
+            # We use counts reference of all errors and not the `@errors` array because it allows
+            # us to go beyond the whole errors storage limit
+            @counts.values.sum
           end
 
           # @return [StandardError, nil] last error that occurred or nil if no errors

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -98,6 +98,17 @@ module Karafka
             :mark_as_consumed
           end
 
+          # The first (lowest) message we want to mark as consumed in marking. By default it uses
+          #   same position as cursor in case user wants to mark same message as consumed as the
+          #   one on which cursor action is applied.
+          # @return [Karafka::Messages::Message, nil] cursor marking message or nil if none
+          # @note It should not return position in time format, only numerical offset
+          def marking_cursor
+            return nil unless active?
+
+            applied.map(&:marking_cursor).compact.min_by(&:offset)
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -42,9 +42,11 @@ module Karafka
             @applied
           end
 
-          # @return [Integer] default timeout for pausing (if applicable)
+          # @return [Integer, nil] default timeout for pausing (if applicable) or nil if not
+          # @note Please do not return `0` when your filter is not pausing as it may interact
+          #   with other filters that want to pause.
           def timeout
-            0
+            nil
           end
 
           # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
@@ -58,6 +60,12 @@ module Karafka
           def marking_method
             :mark_as_consumed
           end
+
+          # @return [Karafka::Messages::Message, nil] cursor message for marking or nil if no
+          #   marking
+          def marking_cursor
+            cursor
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -36,6 +36,11 @@ module Karafka
               too_old
             end
           end
+
+          # @return [nil] this filter does not deal with timeouts
+          def timeout
+            nil
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -54,9 +54,9 @@ module Karafka
             @applied = true
           end
 
-          # @return [Integer] ms timeout in case of pause
+          # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
           def timeout
-            @cursor && applied? ? PAUSE_TIMEOUT : 0
+            @cursor && applied? ? PAUSE_TIMEOUT : nil
           end
 
           # Pause when we had to back-off or skip if delay is not needed

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -37,6 +37,11 @@ module Karafka
 
             messages.delete_if { |message| marked.include?(message.offset) }
           end
+
+          # @return [nil] This filter does not deal with pausing, so timeout is always nil
+          def timeout
+            nil
+          end
         end
       end
     end

data/lib/karafka/pro/processing/parallel_segments/filters/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        # Module for filters injected into the processing pipeline of each of the topics used
+        # within the parallel segmented consumer groups
+        module Filters
+          # Base class for filters for parallel segments that deal with different feature scenarios
+          class Base < Processing::Filters::Base
+            # @param segment_id [Integer] numeric id of the parallel segment group to use with the
+            #   partitioner and reducer for segment matching comparison
+            # @param partitioner [Proc]
+            # @param reducer [Proc]
+            def initialize(segment_id:, partitioner:, reducer:)
+              super()
+
+              @segment_id = segment_id
+              @partitioner = partitioner
+              @reducer = reducer
+            end
+
+            private
+
+            # @param message [Karafka::Messages::Message] received message
+            # @return [String, Numeric] segment assignment key
+            def partition(message)
+              @partitioner.call(message)
+            rescue StandardError => e
+              # This should not happen. If you are seeing this it means your partitioner code
+              # failed and raised an error. We highly recommend mitigating partitioner level errors
+              # on the user side because this type of collapse should be considered a last resort
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message: message,
+                type: 'parallel_segments.partitioner.error'
+              )
+
+              :failure
+            end
+
+            # @param message_segment_key [String, Numeric] segment key to pass to the reducer
+            # @return [Integer] segment assignment of a given message
+            def reduce(message_segment_key)
+              # Assign to segment 0 always in case of failures in partitioner
+              # This is a fail-safe
+              return 0 if message_segment_key == :failure
+
+              @reducer.call(message_segment_key)
+            rescue StandardError => e
+              # @see `#partition` method error handling doc
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message_segment_key: message_segment_key,
+                type: 'parallel_segments.reducer.error'
+              )
+
+              0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/default.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components namespace for parallel segments feature
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments with automatic offset management. Handles
+          # message distribution and ensures proper offset management when messages are filtered
+          # out during the distribution process.
+          #
+          # When operating in automatic offset management mode, this filter takes care of marking
+          # offsets of messages that were filtered out during the distribution process to maintain
+          # proper offset progression.
+          #
+          # @note This is the default filter that should be used when manual offset management
+          #   is not enabled. For manual offset management scenarios use the Mom filter instead.
+          class Default < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+              @all_filtered = false
+              @cursor = messages.first
+
+              # Keep track of how many messages we had initially
+              initial_size = messages.size
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+
+                # Remove the message if it doesn't belong to our group
+                remove = target_segment != @segment_id
+
+                if remove
+                  @cursor = message
+                  @applied = true
+                end
+
+                remove
+              end
+
+              # If all messages were filtered out, we want to mark them as consumed
+              @all_filtered = messages.empty? && initial_size.positive?
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] true if we should mark as consumed (when all were filtered)
+            def mark_as_consumed?
+              @all_filtered
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+
+            # Only return cursor if we wanted to mark as consumed in case all was filtered.
+            # Otherwise it could interfere with other filters
+            def cursor
+              @all_filtered ? @cursor : nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments when manual offset management (mom) is
+          # enabled. Provides message distribution without any post-filtering offset state
+          # management as it is fully user-based.
+          #
+          # Since with manual offset management we need to ensure that offsets are never marked
+          # even in cases where all data in a batch is filtered out.
+          #
+          # This separation allows for cleaner implementation and easier debugging of each flow.
+          #
+          # @note This filter should be used only when manual offset management is enabled.
+          #   For automatic offset management scenarios use the regular filter instead.
+          class Mom < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic without any offset marking
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+                # Remove the message if it doesn't belong to our segment
+                remove = target_segment != @segment_id
+
+                @applied = true if remove
+
+                remove
+              end
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] false, as mom mode never marks as consumed automatically
+            def mark_as_consumed?
+              false
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/partitioner.rb

@@ -38,19 +38,7 @@ module Karafka
             # reduce the whole set into one partition and emit error. This should still allow for
             # user flow but should mitigate damages by not virtualizing
             begin
-              groupings = messages.group_by do |msg|
-                # We need to reduce it to the max concurrency, so the group_id is not a direct
-                # effect of the end user action. Otherwise the persistence layer for consumers
-                # would cache it forever and it would cause memory leaks
-                #
-                # This also needs to be consistent because the aggregation here needs to warrant,
-                # that the same partitioned message will always be assigned to the same virtual
-                # partition. Otherwise in case of a window aggregation with VP spanning across
-                # several polls, the data could not be complete.
-                vps.reducer.call(
-                  vps.partitioner.call(msg)
-                )
-              end
+              groupings = vps.distributor.call(messages)
             rescue StandardError => e
               # This should not happen. If you are seeing this it means your partitioner code
               # failed and raised an error. We highly recommend mitigating partitioner level errors