karafka 2.4.18 → 2.5.0.beta2

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -54,9 +54,9 @@ module Karafka
             @applied = true
           end
 
-          # @return [Integer] ms timeout in case of pause
+          # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
           def timeout
-            @cursor && applied? ? PAUSE_TIMEOUT : 0
+            @cursor && applied? ? PAUSE_TIMEOUT : nil
           end
 
           # Pause when we had to back-off or skip if delay is not needed

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -37,6 +37,11 @@ module Karafka
 
             messages.delete_if { |message| marked.include?(message.offset) }
           end
+
+          # @return [nil] This filter does not deal with pausing, so timeout is always nil
+          def timeout
+            nil
+          end
         end
       end
     end

data/lib/karafka/pro/processing/parallel_segments/filters/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        # Module for filters injected into the processing pipeline of each of the topics used
+        # within the parallel segmented consumer groups
+        module Filters
+          # Base class for filters for parallel segments that deal with different feature scenarios
+          class Base < Processing::Filters::Base
+            # @param segment_id [Integer] numeric id of the parallel segment group to use with the
+            #   partitioner and reducer for segment matching comparison
+            # @param partitioner [Proc]
+            # @param reducer [Proc]
+            def initialize(segment_id:, partitioner:, reducer:)
+              super()
+
+              @segment_id = segment_id
+              @partitioner = partitioner
+              @reducer = reducer
+            end
+
+            private
+
+            # @param message [Karafka::Messages::Message] received message
+            # @return [String, Numeric] segment assignment key
+            def partition(message)
+              @partitioner.call(message)
+            rescue StandardError => e
+              # This should not happen. If you are seeing this it means your partitioner code
+              # failed and raised an error. We highly recommend mitigating partitioner level errors
+              # on the user side because this type of collapse should be considered a last resort
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message: message,
+                type: 'parallel_segments.partitioner.error'
+              )
+
+              :failure
+            end
+
+            # @param message_segment_key [String, Numeric] segment key to pass to the reducer
+            # @return [Integer] segment assignment of a given message
+            def reduce(message_segment_key)
+              # Assign to segment 0 always in case of failures in partitioner
+              # This is a fail-safe
+              return 0 if message_segment_key == :failure
+
+              @reducer.call(message_segment_key)
+            rescue StandardError => e
+              # @see `#partition` method error handling doc
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                message_segment_key: message_segment_key,
+                type: 'parallel_segments.reducer.error'
+              )
+
+              0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/default.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Processing components namespace for parallel segments feature
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments with automatic offset management. Handles
+          # message distribution and ensures proper offset management when messages are filtered
+          # out during the distribution process.
+          #
+          # When operating in automatic offset management mode, this filter takes care of marking
+          # offsets of messages that were filtered out during the distribution process to maintain
+          # proper offset progression.
+          #
+          # @note This is the default filter that should be used when manual offset management
+          #   is not enabled. For manual offset management scenarios use the Mom filter instead.
+          class Default < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+              @all_filtered = false
+              @cursor = messages.first
+
+              # Keep track of how many messages we had initially
+              initial_size = messages.size
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+
+                # Remove the message if it doesn't belong to our group
+                remove = target_segment != @segment_id
+
+                if remove
+                  @cursor = message
+                  @applied = true
+                end
+
+                remove
+              end
+
+              # If all messages were filtered out, we want to mark them as consumed
+              @all_filtered = messages.empty? && initial_size.positive?
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] true if we should mark as consumed (when all were filtered)
+            def mark_as_consumed?
+              @all_filtered
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+
+            # Only return cursor if we wanted to mark as consumed in case all was filtered.
+            # Otherwise it could interfere with other filters
+            def cursor
+              @all_filtered ? @cursor : nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module ParallelSegments
+        module Filters
+          # Filter used for handling parallel segments when manual offset management (mom) is
+          # enabled. Provides message distribution without any post-filtering offset state
+          # management as it is fully user-based.
+          #
+          # Since with manual offset management we need to ensure that offsets are never marked
+          # even in cases where all data in a batch is filtered out.
+          #
+          # This separation allows for cleaner implementation and easier debugging of each flow.
+          #
+          # @note This filter should be used only when manual offset management is enabled.
+          #   For automatic offset management scenarios use the regular filter instead.
+          class Mom < Base
+            # Applies the filter to the batch of messages
+            # It removes messages that don't belong to the current parallel segment group
+            # based on the partitioner and reducer logic without any offset marking
+            #
+            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+            #   filter
+            def apply!(messages)
+              @applied = false
+
+              # Filter out messages that don't match our segment group
+              messages.delete_if do |message|
+                message_segment_key = partition(message)
+                # Use the reducer to get the target group for this message
+                target_segment = reduce(message_segment_key)
+                # Remove the message if it doesn't belong to our segment
+                remove = target_segment != @segment_id
+
+                @applied = true if remove
+
+                remove
+              end
+            end
+
+            # @return [Boolean] true if any messages were filtered out
+            def applied?
+              @applied
+            end
+
+            # @return [Boolean] false, as mom mode never marks as consumed automatically
+            def mark_as_consumed?
+              false
+            end
+
+            # @return [nil] Since we do not timeout ever in this filter, we should not return
+            #   any value for it.
+            def timeout
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/partitioner.rb

@@ -38,19 +38,7 @@ module Karafka
             # reduce the whole set into one partition and emit error. This should still allow for
             # user flow but should mitigate damages by not virtualizing
             begin
-              groupings = messages.group_by do |msg|
-                # We need to reduce it to the max concurrency, so the group_id is not a direct
-                # effect of the end user action. Otherwise the persistence layer for consumers
-                # would cache it forever and it would cause memory leaks
-                #
-                # This also needs to be consistent because the aggregation here needs to warrant,
-                # that the same partitioned message will always be assigned to the same virtual
-                # partition. Otherwise in case of a window aggregation with VP spanning across
-                # several polls, the data could not be complete.
-                vps.reducer.call(
-                  vps.partitioner.call(msg)
-                )
-              end
+              groupings = vps.distributor.call(messages)
             rescue StandardError => e
               # This should not happen. If you are seeing this it means your partitioner code
               # failed and raised an error. We highly recommend mitigating partitioner level errors

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -20,16 +20,16 @@ module Karafka
 
           # Pipes given message to the provided topic with expected details. Useful for
           # pass-through operations where deserialization is not needed. Upon usage it will include
-          # all the original headers + meta headers about the source of message.
+          # all the source headers + meta headers about the source of message.
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           #
           # @note It will NOT deserialize the payload so it is fast
           #
           # @note We assume that there can be different number of partitions in the target topic,
-          #   this is why we use `key` based on the original topic key and not the partition id.
-          #   This will not utilize partitions beyond the number of partitions of original topic,
+          #   this is why we use `key` based on the source topic key and not the partition id.
+          #   This will not utilize partitions beyond the number of partitions of source topic,
           #   but will accommodate for topics with less partitions.
           def pipe_async(topic:, message:)
             produce_async(
@@ -40,7 +40,7 @@ module Karafka
           # Sync version of pipe for one message
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @see [#pipe_async]
           def pipe_sync(topic:, message:)
             produce_sync(
@@ -51,7 +51,7 @@ module Karafka
           # Async multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -66,7 +66,7 @@ module Karafka
           # Sync multi-message pipe
           #
           # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] original messages to pipe
+          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
           #
           # @note If transactional producer in use and dispatch is not wrapped with a transaction,
           #   it will automatically wrap the dispatch with a transaction
@@ -81,7 +81,7 @@ module Karafka
           private
 
           # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] original message to pipe
+          # @param message [Karafka::Messages::Message] source message to pipe
           # @return [Hash] hash with message to pipe.
           #
           # @note If you need to alter this, please define the `#enhance_pipe_message` method
@@ -90,17 +90,17 @@ module Karafka
               topic: topic,
               payload: message.raw_payload,
               headers: message.raw_headers.merge(
-                'original_topic' => message.topic,
-                'original_partition' => message.partition.to_s,
-                'original_offset' => message.offset.to_s,
-                'original_consumer_group' => self.topic.consumer_group.id
+                'source_topic' => message.topic,
+                'source_partition' => message.partition.to_s,
+                'source_offset' => message.offset.to_s,
+                'source_consumer_group' => self.topic.consumer_group.id
               )
             }
 
             # Use a key only if key was provided
             if message.raw_key
               pipe_message[:key] = message.raw_key
-            # Otherwise pipe creating a key that will assign it based on the original partition
+            # Otherwise pipe creating a key that will assign it based on the source partition
             # number
             else
               pipe_message[:key] = message.partition.to_s

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -38,7 +38,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -44,7 +44,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -36,7 +36,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   mark_as_consumed(last_group_message) unless revoked?
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -43,7 +43,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(seek_offset, false) unless revoked?
+                  seek(seek_offset, false, reset_offset: false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/default.rb

@@ -55,14 +55,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, true)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed(message, offset_metadata)
                        end
@@ -92,14 +97,19 @@ module Karafka
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
               return true if seek_offset.nil?
-              # Ignore earlier offsets than the one we already committed
-              return true if seek_offset > message.offset
+              # Ignore if it is the same offset as the one that is marked currently
+              # We ignore second marking because it changes nothing and in case of people using
+              # metadata storage but with automatic offset marking, this would cause metadata to be
+              # erased by automatic marking
+              return true if (seek_offset - 1) == message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
               # artificially created transaction
               stored = if producer.transactional?
                          mark_with_transaction(message, offset_metadata, false)
+                       elsif @_transactional_marking
+                         raise Errors::NonTransactionalMarkingAttemptError
                        else
                          client.mark_as_consumed!(message, offset_metadata)
                        end
@@ -143,6 +153,7 @@ module Karafka
               self.producer = active_producer
 
               transaction_started = false
+              transaction_completed = false
 
               # Prevent from nested transactions. It would not make any sense
               raise Errors::TransactionAlreadyInitializedError if @_in_transaction
@@ -159,6 +170,12 @@ module Karafka
                 # transaction. We do it only for transactions that contain offset management as for
                 # producer only, this is not relevant.
                 raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+
+                # If we do not reach this, we should not move seek offsets because it means that
+                # either an error occured or transaction was aborted.
+                # In case of error, it will bubble up so no issue but in case of abort, while we
+                # do not reach this place, the code will continue
+                transaction_completed = true
               end
 
               @_in_transaction = false
@@ -180,8 +197,13 @@ module Karafka
               #     to mimic this
               #   - Complex strategies like VPs can use this in VPs to mark in parallel without
               #     having to redefine the transactional flow completely
-              @_transaction_marked.each do |marking|
-                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              #
+              # @note This should be applied only if transaction did not error and if it was not
+              #   aborted.
+              if transaction_completed
+                @_transaction_marked.each do |marking|
+                  marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+                end
               end
 
               true
@@ -213,6 +235,9 @@ module Karafka
               offset_metadata
             )
 
+            # This one is long lived and used to make sure, that users do not mix transactional
+            # marking with non-transactional. When this happens we should raise error
+            @_transactional_marking = true
             @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
@@ -252,8 +277,11 @@ module Karafka
             # seek offset can be nil only in case `#seek` was invoked with offset reset request
             # In case like this we ignore marking
             return true if seek_offset.nil?
-            # Ignore earlier offsets than the one we already committed
-            return true if seek_offset > message.offset
+            # Ignore if it is the same offset as the one that is marked currently
+            # We ignore second marking because it changes nothing and in case of people using
+            # metadata storage but with automatic offset marking, this would cause metadata to be
+            # erased by automatic marking
+            return true if (seek_offset - 1) == message.offset
             return false if revoked?
 
             # If we have already marked this successfully in a transaction that was running

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -145,19 +145,19 @@ module Karafka
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
-              original_partition = skippable_message.partition.to_s
+              source_partition = skippable_message.partition.to_s
 
               dlq_message = {
-                topic: topic.dead_letter_queue.topic,
-                key: original_partition,
+                topic: @_dispatch_to_dlq_topic || topic.dead_letter_queue.topic,
+                key: source_partition,
                 payload: skippable_message.raw_payload,
                 headers: skippable_message.raw_headers.merge(
-                  'original_topic' => topic.name,
-                  'original_partition' => original_partition,
-                  'original_offset' => skippable_message.offset.to_s,
-                  'original_consumer_group' => topic.consumer_group.id,
-                  'original_key' => skippable_message.raw_key.to_s,
-                  'original_attempts' => attempt.to_s
+                  'source_topic' => topic.name,
+                  'source_partition' => source_partition,
+                  'source_offset' => skippable_message.offset.to_s,
+                  'source_consumer_group' => topic.consumer_group.id,
+                  'source_key' => skippable_message.raw_key.to_s,
+                  'source_attempts' => attempt.to_s
                 )
               }
 
@@ -205,7 +205,7 @@ module Karafka
             # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
             # In case of `:retry` always `#retry_after_pause` is applied
             def apply_dlq_flow
-              flow = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
+              flow, target_topic = topic.dead_letter_queue.strategy.call(errors_tracker, attempt)
 
               case flow
               when :retry
@@ -216,6 +216,8 @@ module Karafka
                 @_dispatch_to_dlq = false
               when :dispatch
                 @_dispatch_to_dlq = true
+                # Use custom topic if it was returned from the strategy
+                @_dispatch_to_dlq_topic = target_topic || topic.dead_letter_queue.topic
               else
                 raise Karafka::UnsupportedCaseError, flow
               end
@@ -227,6 +229,8 @@ module Karafka
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
               pause(seek_offset, nil, false)
+            ensure
+              @_dispatch_to_dlq_topic = nil
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -40,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -35,7 +35,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -31,7 +31,9 @@ module Karafka
 
                   mark_as_consumed(last_group_message) unless revoked?
                   # We should not overwrite user manual seel request with our seek
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -30,7 +30,7 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false)
+                    seek(last_group_message.offset + 1, false, reset_offset: false)
                   end
 
                   resume

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -67,7 +67,7 @@ module Karafka
               if filter.mark_as_consumed?
                 send(
                   filter.marking_method,
-                  filter.cursor
+                  filter.marking_cursor
                 )
               end
 

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -43,7 +43,10 @@ module Karafka
                   return if coordinator.manual_pause?
 
                   mark_as_consumed(last_group_message) unless revoked?
-                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
+
+                  unless revoked? || coordinator.manual_seek?
+                    seek(seek_offset, false, reset_offset: false)
+                  end
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -40,7 +40,7 @@ module Karafka
                   elsif !revoked? && !coordinator.manual_seek?
                     # If not revoked and not throttled, we move to where we were suppose to and
                     # resume
-                    seek(seek_offset, false)
+                    seek(seek_offset, false, reset_offset: false)
                     resume
                   else
                     resume