karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/mom.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module ParallelSegments
+          module Filters
+            # Filter used for handling parallel segments when manual offset management (mom) is
+            # enabled. Provides message distribution without any post-filtering offset state
+            # management as it is fully user-based.
+            #
+            # Since with manual offset management we need to ensure that offsets are never marked
+            # even in cases where all data in a batch is filtered out.
+            #
+            # This separation allows for cleaner implementation and easier debugging of each flow.
+            #
+            # @note This filter should be used only when manual offset management is enabled.
+            #   For automatic offset management scenarios use the regular filter instead.
+            class Mom < Base
+              # Applies the filter to the batch of messages
+              # It removes messages that don't belong to the current parallel segment group
+              # based on the partitioner and reducer logic without any offset marking
+              #
+              # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+              #   filter
+              def apply!(messages)
+                @applied = false
+
+                # Filter out messages that don't match our segment group
+                messages.delete_if do |message|
+                  message_segment_key = partition(message)
+                  # Use the reducer to get the target group for this message
+                  target_segment = reduce(message_segment_key)
+                  # Remove the message if it doesn't belong to our segment
+                  remove = target_segment != @segment_id
+
+                  @applied = true if remove
+
+                  remove
+                end
+              end
+
+              # @return [Boolean] true if any messages were filtered out
+              def applied?
+                @applied
+              end
+
+              # @return [Boolean] false, as mom mode never marks as consumed automatically
+              def mark_as_consumed?
+                false
+              end
+
+              # @return [nil] Since we do not timeout ever in this filter, we should not return
+              #   any value for it.
+              def timeout
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/partitioner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Pro partitioner that can distribute work based on the virtual partitioner settings
+        class Partitioner < Karafka::Processing::ConsumerGroups::Partitioner
+          # @param topic [String] topic name
+          # @param messages [Array<Karafka::Messages::Message>] karafka messages
+          # @param coordinator [Karafka::Pro::Processing::ConsumerGroups::Coordinator] processing coordinator that
+          #   will be used with those messages
+          # @yieldparam [Integer] group id
+          # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
+          def call(topic, messages, coordinator)
+            ktopic = @subscription_group.topics.find(topic)
+
+            vps = ktopic.virtual_partitions
+
+            # We only partition work if we have:
+            # - a virtual partitioner
+            # - more than one thread to process the data
+            # - collective is not collapsed via coordinator
+            # - none of the partitioner executions raised an error
+            #
+            # With one thread it is not worth partitioning the work as the work itself will be
+            # assigned to one thread (pointless work)
+            #
+            # We collapse the partitioning on errors because we "regain" full ordering on a batch
+            # that potentially contains the data that caused the error.
+            #
+            # This is great because it allows us to run things without the parallelization that adds
+            # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
+            if vps.active? && vps.max_partitions > 1 && !coordinator.collapsed?
+              # If we cannot virtualize even one message from a given batch due to user errors, we
+              # reduce the whole set into one partition and emit error. This should still allow for
+              # user flow but should mitigate damages by not virtualizing
+              begin
+                groupings = vps.distributor.call(messages)
+              rescue => e
+                # This should not happen. If you are seeing this it means your partitioner code
+                # failed and raised an error. We highly recommend mitigating partitioner level errors
+                # on the user side because this type of collapse should be considered a last resort
+                Karafka.monitor.instrument(
+                  "error.occurred",
+                  caller: self,
+                  error: e,
+                  messages: messages,
+                  type: "virtual_partitions.partitioner.error"
+                )
+
+                groupings = { 0 => messages }
+              end
+
+              groupings.each do |key, messages_group|
+                yield(key, messages_group)
+              end
+            else
+              # When no virtual partitioner, works as regular one
+              yield(0, messages)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/periodic_job/consumer.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Namespace for periodic jobs related processing APIs
+        module PeriodicJob
+          # Consumer extra methods useful only when periodic jobs are in use
+          module Consumer
+            class << self
+              # Defines an empty `#tick` method if not present
+              #
+              # We define it that way due to our injection strategy flow.
+              #
+              # @param consumer_singleton_class [Karafka::BaseConsumer] consumer singleton class
+              #   that is being enriched with periodic jobs API
+              def included(consumer_singleton_class)
+                # Do not define empty tick method on consumer if it already exists
+                # We only define it when it does not exist to have empty periodic ticking
+                #
+                # We need to check both cases (public and private) since user is not expected to
+                # have this method public
+                return if consumer_singleton_class.method_defined?(:tick)
+                return if consumer_singleton_class.private_instance_methods.include?(:tick)
+
+                # Create empty ticking method
+                consumer_singleton_class.class_eval do
+                  def tick
+                  end
+                end
+              end
+            end
+
+            # Runs the on-schedule tick periodic operations
+            # This method is an alias but is part of the naming convention used for other flows, this
+            # is why we do not reference the `handle_before_schedule_tick` directly
+            def on_before_schedule_tick
+              handle_before_schedule_tick
+            end
+
+            # Used by the executor to trigger consumer tick
+            # @private
+            def on_tick
+              handle_tick
+            rescue => e
+              Karafka.monitor.instrument(
+                "error.occurred",
+                error: e,
+                caller: self,
+                type: "consumer.tick.error"
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/piping/consumer.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # All code needed for messages piping in Karafka
+        module Piping
+          # Consumer piping functionality
+          #
+          # It provides way to pipe data in a consistent way with extra traceability headers similar
+          # to those in the enhanced DLQ.
+          module Consumer
+            # Empty hash to save on memory allocations
+            EMPTY_HASH = {}.freeze
+
+            private_constant :EMPTY_HASH
+
+            # Pipes given message to the provided topic with expected details. Useful for
+            # pass-through operations where deserialization is not needed. Upon usage it will include
+            # all the source headers + meta headers about the source of message.
+            #
+            # @param topic [String, Symbol] where we want to send the message
+            # @param message [Karafka::Messages::Message] source message to pipe
+            #
+            # @note It will NOT deserialize the payload so it is fast
+            #
+            # @note We assume that there can be different number of partitions in the target topic,
+            #   this is why we use `key` based on the source topic key and not the partition id.
+            #   This will not utilize partitions beyond the number of partitions of source topic,
+            #   but will accommodate for topics with less partitions.
+            def pipe_async(topic:, message:)
+              produce_async(
+                build_pipe_message(topic: topic, message: message)
+              )
+            end
+
+            # Sync version of pipe for one message
+            #
+            # @param topic [String, Symbol] where we want to send the message
+            # @param message [Karafka::Messages::Message] source message to pipe
+            # @see [#pipe_async]
+            def pipe_sync(topic:, message:)
+              produce_sync(
+                build_pipe_message(topic: topic, message: message)
+              )
+            end
+
+            # Async multi-message pipe
+            #
+            # @param topic [String, Symbol] where we want to send the message
+            # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
+            #
+            # @note If transactional producer in use and dispatch is not wrapped with a transaction,
+            #   it will automatically wrap the dispatch with a transaction
+            def pipe_many_async(topic:, messages:)
+              messages = messages.map do |message|
+                build_pipe_message(topic: topic, message: message)
+              end
+
+              produce_many_async(messages)
+            end
+
+            # Sync multi-message pipe
+            #
+            # @param topic [String, Symbol] where we want to send the message
+            # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
+            #
+            # @note If transactional producer in use and dispatch is not wrapped with a transaction,
+            #   it will automatically wrap the dispatch with a transaction
+            def pipe_many_sync(topic:, messages:)
+              messages = messages.map do |message|
+                build_pipe_message(topic: topic, message: message)
+              end
+
+              produce_many_sync(messages)
+            end
+
+            private
+
+            # @param topic [String, Symbol] where we want to send the message
+            # @param message [Karafka::Messages::Message] source message to pipe
+            # @return [Hash] hash with message to pipe.
+            #
+            # @note If you need to alter this, please define the `#enhance_pipe_message` method
+            def build_pipe_message(topic:, message:)
+              pipe_message = {
+                topic: topic,
+                payload: message.raw_payload,
+                headers: message.raw_headers.merge(
+                  "source_topic" => message.topic,
+                  "source_partition" => message.partition.to_s,
+                  "source_offset" => message.offset.to_s,
+                  "source_consumer_group" => self.topic.group.id
+                )
+              }
+
+              # Use a key only if key was provided
+              pipe_message[:key] = if message.raw_key
+                message.raw_key
+              # Otherwise pipe creating a key that will assign it based on the source partition
+              # number
+              else
+                message.partition.to_s
+              end
+
+              # Optional method user can define in consumer to enhance the dlq message hash with
+              # some extra details if needed or to replace payload, etc
+              if respond_to?(:enhance_pipe_message, true)
+                enhance_pipe_message(
+                  pipe_message,
+                  message
+                )
+              end
+
+              pipe_message
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # DLQ enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            module DlqFtrLrjMom
+              include Strategies::Aj::FtrMom
+              include Strategies::Aj::DlqMom
+              include Strategies::Aj::LrjMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                filtering
+                long_running_job
+                manual_offset_management
+              ].freeze
+
+              # This strategy assumes we do not early break on shutdown as it has VP
+              def handle_after_consume
+                coordinator.on_finished do
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked?
+                      # no need to check for manual seek because AJ consumer is internal and
+                      # fully controlled by us
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Strategies
+          module Aj
+            # ActiveJob enabled
+            # DLQ enabled
+            # Filtering enabled
+            # Long-Running Job enabled
+            # Manual offset management enabled
+            # Virtual Partitions enabled
+            module DlqFtrLrjMomVp
+              include Strategies::Aj::FtrMom
+              include Strategies::Aj::DlqMomVp
+              include Strategies::Aj::LrjMom
+
+              # Features for this strategy
+              FEATURES = %i[
+                active_job
+                dead_letter_queue
+                filtering
+                long_running_job
+                manual_offset_management
+                virtual_partitions
+              ].freeze
+
+              # This strategy assumes we do not early break on shutdown as it has VP
+              def handle_after_consume
+                coordinator.on_finished do |last_group_message|
+                  if coordinator.success?
+                    coordinator.pause_tracker.reset
+
+                    # Since we have VP here we do not commit intermediate offsets and need to commit
+                    # them here. We do commit in collapsed mode but this is generalized.
+                    mark_as_consumed(last_group_message) unless revoked?
+
+                    if coordinator.filtered? && !revoked?
+                      handle_post_filtering
+
+                      # :seek and :pause are fully handled by handle_post_filtering
+                      # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
+                      return unless coordinator.filter.action == :skip
+                    elsif !revoked?
+                      # no need to check for manual seek because AJ consumer is internal and
+                      # fully controlled by us
+                      seek(seek_offset, false, reset_offset: false)
+                    end
+
+                    resume
+                  else
+                    apply_dlq_flow do
+                      skippable_message, = find_skippable_message
+                      dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                      mark_dispatched_to_dlq(skippable_message)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end