karafka 2.4.18 → 2.5.0

data/lib/karafka/pro/encryption.rb

@@ -22,7 +22,10 @@ module Karafka
 
         # @param config [Karafka::Core::Configurable::Node] root node config
         def post_setup(config)
-          Encryption::Contracts::Config.new.validate!(config.to_h)
+          Encryption::Contracts::Config.new.validate!(
+            config.to_h,
+            scope: %w[config]
+          )
 
           # Don't inject extra components if encryption is not active
           return unless config.encryption.active
@@ -33,6 +36,14 @@ module Karafka
           # Encryption for WaterDrop
           config.producer.middleware.append(Messages::Middleware.new)
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -50,7 +50,7 @@ module Karafka
           partition = messages.metadata.partition
 
           samples = @processing_times[topic][partition]
-          samples << event[:time] / messages.count
+          samples << event[:time] / messages.size
 
           return unless samples.size > SAMPLES_COUNT
 

data/lib/karafka/pro/iterator/expander.rb

@@ -21,8 +21,10 @@ module Karafka
       # - { 'topic1' => 100 } - means we run all partitions from the offset 100
       # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
       # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
-      # - { 'topic1' => { 1 => true } } - will pick first offset not consumed on this CG for p 1
-      # - { 'topic1' => true } - will pick first offset not consumed on this CG for all p
+      # - { 'topic1' => { 1 => true } } - will pick first offset on this CG for partition 1
+      # - { 'topic1' => true } - will pick first offset for all partitions
+      # - { 'topic1' => :earliest } - will pick earliest offset for all partitions
+      # - { 'topic1' => :latest } - will pick latest (high-watermark) for all partitions
       class Expander
         # Expands topics to which we want to subscribe with partitions information in case this
         # info is not provided.
@@ -80,7 +82,7 @@ module Karafka
             .find { |topic| topic.fetch(:topic_name) == name }
             .tap { |topic| topic || raise(Errors::TopicNotFoundError, name) }
             .fetch(:partitions)
-            .count
+            .size
         end
       end
     end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -14,6 +14,11 @@ module Karafka
       # This builder resolves that and builds a tpl to which we can safely subscribe the way
       # we want it.
       class TplBuilder
+        # Supported named offset positions that we can reference via their name
+        SUPPORTED_NAMED_POSITIONS = %w[earliest latest].freeze
+
+        private_constant :SUPPORTED_NAMED_POSITIONS
+
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
@@ -28,6 +33,7 @@ module Karafka
           resolve_partitions_with_exact_offsets
           resolve_partitions_with_negative_offsets
           resolve_partitions_with_time_offsets
+          resolve_partitions_with_named_offsets
           resolve_partitions_with_cg_expectations
 
           # Final tpl with all the data
@@ -143,6 +149,23 @@ module Karafka
           end
         end
 
+        # If we get named offsets, we can just remap them to librdkafka special offset positions
+        def resolve_partitions_with_named_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Skip offsets that do not match our named expectations
+              named_offset = offset.to_s
+
+              next unless SUPPORTED_NAMED_POSITIONS.include?(named_offset)
+
+              @mapped_topics[name][partition] = -1 if named_offset == 'latest'
+              @mapped_topics[name][partition] = -2 if named_offset == 'earliest'
+            end
+          end
+        end
+
         # Fetches last used offsets for those partitions for which we want to consume from last
         # moment where given consumer group has finished
         # This is indicated by given partition value being set to `true`.

data/lib/karafka/pro/loader.rb

@@ -60,6 +60,15 @@ module Karafka
           Processing::SubscriptionGroupsCoordinator.instance
         end
 
+        # Runs operations needed after fork in swarm for features that need it
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer] pre fork producer instance that may be
+        #   needed to be replaced with newly changed one post-fork.
+        def post_fork(config, pre_fork_producer)
+          features.each { |feature| feature.post_fork(config, pre_fork_producer) }
+        end
+
         private
 
         # @return [Array<Module>] extra non-routing related pro features and routing components
@@ -84,6 +93,7 @@ module Karafka
           icfg.connection.manager = Connection::Manager.new
 
           icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.errors_tracker_class = Processing::Coordinators::ErrorsTracker
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue

data/lib/karafka/pro/processing/coordinator.rb

@@ -10,6 +10,9 @@ module Karafka
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
         extend Forwardable
+        include Helpers::ConfigImporter.new(
+          errors_tracker_class: %i[internal processing errors_tracker_class]
+        )
 
         def_delegators :@collapser, :collapsed?, :collapse_until!
 
@@ -20,7 +23,7 @@ module Karafka
           super
 
           @executed = []
-          @errors_tracker = Coordinators::ErrorsTracker.new
+          @errors_tracker = errors_tracker_class.new(topic, partition)
           @flow_mutex = Mutex.new
           # Lock for user code synchronization
           # We do not want to mix coordinator lock with the user lock not to create cases where

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -13,25 +13,52 @@ module Karafka
         class ErrorsTracker
           include Enumerable
 
+          # @return [Karafka::Routing::Topic] topic of this error tracker
+          attr_reader :topic
+
+          # @return [Integer] partition of this error tracker
+          attr_reader :partition
+
+          # @return [Hash]
+          attr_reader :counts
+
+          # @return [String]
+          attr_reader :trace_id
+
           # Max errors we keep in memory.
           # We do not want to keep more because for DLQ-less this would cause memory-leaks.
+          # We do however count per class for granular error counting
           STORAGE_LIMIT = 100
 
           private_constant :STORAGE_LIMIT
 
-          def initialize
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer]
+          # @param limit [Integer] max number of errors we want to keep for reference when
+          #   implementing custom error handling.
+          # @note `limit` does not apply to the counts. They will work beyond the number of errors
+          #   occurring
+          def initialize(topic, partition, limit: STORAGE_LIMIT)
             @errors = []
+            @counts = Hash.new { |hash, key| hash[key] = 0 }
+            @topic = topic
+            @partition = partition
+            @limit = limit
+            @trace_id = SecureRandom.uuid
           end
 
           # Clears all the errors
           def clear
             @errors.clear
+            @counts.clear
           end
 
           # @param error [StandardError] adds the error to the tracker
           def <<(error)
-            @errors.shift if @errors.size >= STORAGE_LIMIT
+            @errors.shift if @errors.size >= @limit
             @errors << error
+            @counts[error.class] += 1
+            @trace_id = SecureRandom.uuid
           end
 
           # @return [Boolean] is the error tracker empty
@@ -41,7 +68,9 @@ module Karafka
 
           # @return [Integer] number of elements
           def size
-            count
+            # We use counts reference of all errors and not the `@errors` array because it allows
+            # us to go beyond the whole errors storage limit
+            @counts.values.sum
           end
 
           # @return [StandardError, nil] last error that occurred or nil if no errors

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -98,6 +98,17 @@ module Karafka
             :mark_as_consumed
           end
 
+          # The first (lowest) message we want to mark as consumed in marking. By default it uses
+          #   same position as cursor in case user wants to mark same message as consumed as the
+          #   one on which cursor action is applied.
+          # @return [Karafka::Messages::Message, nil] cursor marking message or nil if none
+          # @note It should not return position in time format, only numerical offset
+          def marking_cursor
+            return nil unless active?
+
+            applied.map(&:marking_cursor).compact.min_by(&:offset)
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -42,9 +42,11 @@ module Karafka
             @applied
           end
 
-          # @return [Integer] default timeout for pausing (if applicable)
+          # @return [Integer, nil] default timeout for pausing (if applicable) or nil if not
+          # @note Please do not return `0` when your filter is not pausing as it may interact
+          #   with other filters that want to pause.
           def timeout
-            0
+            nil
           end
 
           # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
@@ -58,6 +60,12 @@ module Karafka
           def marking_method
             :mark_as_consumed
           end
+
+          # @return [Karafka::Messages::Message, nil] cursor message for marking or nil if no
+          #   marking
+          def marking_cursor
+            cursor
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -36,6 +36,11 @@ module Karafka
               too_old
             end
           end
+
+          # @return [nil] this filter does not deal with timeouts
+          def timeout
+            nil
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -54,9 +54,9 @@ module Karafka
             @applied = true
           end
 
-          # @return [Integer] ms timeout in case of pause
+          # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
           def timeout
-            @cursor && applied? ? PAUSE_TIMEOUT : 0
+            @cursor && applied? ? PAUSE_TIMEOUT : nil
           end
 
           # Pause when we had to back-off or skip if delay is not needed

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -37,6 +37,11 @@ module Karafka
 
             messages.delete_if { |message| marked.include?(message.offset) }
           end
+
+          # @return [nil] This filter does not deal with pausing, so timeout is always nil
+          def timeout
+            nil
+          end
         end
       end
     end

data/lib/karafka/pro/processing/parallel_segments/filters/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        # Module for filters injected into the processing pipeline of each of the topics used
+        # within the parallel segmented consumer groups
+        module Filters
+          # Base class for filters for parallel segments that deal with different feature scenarios
+          class Base < Processing::Filters::Base
+            # @param segment_id [Integer] numeric id of the parallel segment group to use with the
+            #   partitioner and reducer for segment matching comparison
+            # @param partitioner [Proc]
+            # @param reducer [Proc]
+            def initialize(segment_id:, partitioner:, reducer:)
+              super()
+
+              @segment_id = segment_id
+              @partitioner = partitioner
+              @reducer = reducer
+            end
+
+            private
+
+            # @param message [Karafka::Messages::Message] received message
+            # @return [String, Numeric] segment assignment key
+            def partition(message)
+              @partitioner.call(message)
+            rescue StandardError => e
+              # This should not happen. If you are seeing this it means your partitioner code
+              # failed and raised an error. We highly recommend mitigating partitioner level errors
+              # on the user side because this type of collapse should be considered a last resort
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message: message,
+                type: 'parallel_segments.partitioner.error'
+              )
+
+              :failure
+            end
+
+            # @param message_segment_key [String, Numeric] segment key to pass to the reducer
+            # @return [Integer] segment assignment of a given message
+            def reduce(message_segment_key)
+              # Assign to segment 0 always in case of failures in partitioner
+              # This is a fail-safe
+              return 0 if message_segment_key == :failure
+
+              @reducer.call(message_segment_key)
+            rescue StandardError => e
+              # @see `#partition` method error handling doc
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message_segment_key: message_segment_key,
+                type: 'parallel_segments.reducer.error'
+              )
+
+              0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/default.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components namespace for parallel segments feature
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments with automatic offset management. Handles
+          # message distribution and ensures proper offset management when messages are filtered
+          # out during the distribution process.
+          #
+          # When operating in automatic offset management mode, this filter takes care of marking
+          # offsets of messages that were filtered out during the distribution process to maintain
+          # proper offset progression.
+          #
+          # @note This is the default filter that should be used when manual offset management
+          #   is not enabled. For manual offset management scenarios use the Mom filter instead.
+          class Default < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+              @all_filtered = false
+              @cursor = messages.first
+
+              # Keep track of how many messages we had initially
+              initial_size = messages.size
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+
+                # Remove the message if it doesn't belong to our group
+                remove = target_segment != @segment_id
+
+                if remove
+                  @cursor = message
+                  @applied = true
+                end
+
+                remove
+              end
+
+              # If all messages were filtered out, we want to mark them as consumed
+              @all_filtered = messages.empty? && initial_size.positive?
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] true if we should mark as consumed (when all were filtered)
+            def mark_as_consumed?
+              @all_filtered
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+
+            # Only return cursor if we wanted to mark as consumed in case all was filtered.
+            # Otherwise it could interfere with other filters
+            def cursor
+              @all_filtered ? @cursor : nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments when manual offset management (mom) is
+          # enabled. Provides message distribution without any post-filtering offset state
+          # management as it is fully user-based.
+          #
+          # Since with manual offset management we need to ensure that offsets are never marked
+          # even in cases where all data in a batch is filtered out.
+          #
+          # This separation allows for cleaner implementation and easier debugging of each flow.
+          #
+          # @note This filter should be used only when manual offset management is enabled.
+          #   For automatic offset management scenarios use the regular filter instead.
+          class Mom < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic without any offset marking
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+                # Remove the message if it doesn't belong to our segment
+                remove = target_segment != @segment_id
+
+                @applied = true if remove
+
+                remove
+              end
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] false, as mom mode never marks as consumed automatically
+            def mark_as_consumed?
+              false
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/partitioner.rb

@@ -38,19 +38,7 @@ module Karafka
             # reduce the whole set into one partition and emit error. This should still allow for
             # user flow but should mitigate damages by not virtualizing
             begin
-              groupings = messages.group_by do |msg|
-                # We need to reduce it to the max concurrency, so the group_id is not a direct
-                # effect of the end user action. Otherwise the persistence layer for consumers
-                # would cache it forever and it would cause memory leaks
-                #
-                # This also needs to be consistent because the aggregation here needs to warrant,
-                # that the same partitioned message will always be assigned to the same virtual
-                # partition. Otherwise in case of a window aggregation with VP spanning across
-                # several polls, the data could not be complete.
-                vps.reducer.call(
-                  vps.partitioner.call(msg)
-                )
-              end
+              groupings = vps.distributor.call(messages)
             rescue StandardError => e
               # This should not happen. If you are seeing this it means your partitioner code
               # failed and raised an error. We highly recommend mitigating partitioner level errors

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -20,16 +20,16 @@ module Karafka
 
           # Pipes given message to the provided topic with expected details. Useful for
           # pass-through operations where deserialization is not needed. Upon usage it will include
-          # all the original headers + meta headers about the source of message.
+          # all the source headers + meta headers about the source of message.
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           #
           # @note It will NOT deserialize the payload so it is fast
           #
           # @note We assume that there can be different number of partitions in the target topic,
-          #   this is why we use `key` based on the original topic key and not the partition id.
-          #   This will not utilize partitions beyond the number of partitions of original topic,
+          #   this is why we use `key` based on the source topic key and not the partition id.
+          #   This will not utilize partitions beyond the number of partitions of source topic,
           #   but will accommodate for topics with less partitions.
           def pipe_async(topic:, message:)
             produce_async(
@@ -40,7 +40,7 @@ module Karafka
           # Sync version of pipe for one message
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @see [#pipe_async]
           def pipe_sync(topic:, message:)
             produce_sync(
@@ -51,7 +51,7 @@ module Karafka
           # Async multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -66,7 +66,7 @@ module Karafka
           # Sync multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -81,7 +81,7 @@ module Karafka
           private
 
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @return [Hash] hash with message to pipe.
           #
           # @note If you need to alter this, please define the `#enhance_pipe_message` method
@@ -90,17 +90,17 @@ module Karafka
               topic: topic,
               payload: message.raw_payload,
               headers: message.raw_headers.merge(
-                'original_topic' => message.topic,
-                'original_partition' => message.partition.to_s,
-                'original_offset' => message.offset.to_s,
-                'original_consumer_group' => self.topic.consumer_group.id
+                'source_topic' => message.topic,
+                'source_partition' => message.partition.to_s,
+                'source_offset' => message.offset.to_s,
+                'source_consumer_group' => self.topic.consumer_group.id
               )
             }
 
             # Use a key only if key was provided
             if message.raw_key
               pipe_message[:key] = message.raw_key
-            # Otherwise pipe creating a key that will assign it based on the original partition
+            # Otherwise pipe creating a key that will assign it based on the source partition
             # number
             else
               pipe_message[:key] = message.partition.to_s

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -44,7 +44,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -36,7 +36,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   mark_as_consumed(last_group_message) unless revoked?
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else