karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -1,149 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # All code needed for messages piping in Karafka
-      module Piping
-        # Consumer piping functionality
-        #
-        # It provides way to pipe data in a consistent way with extra traceability headers similar
-        # to those in the enhanced DLQ.
-        module Consumer
-          # Empty hash to save on memory allocations
-          EMPTY_HASH = {}.freeze
-
-          private_constant :EMPTY_HASH
-
-          # Pipes given message to the provided topic with expected details. Useful for
-          # pass-through operations where deserialization is not needed. Upon usage it will include
-          # all the source headers + meta headers about the source of message.
-          #
-          # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] source message to pipe
-          #
-          # @note It will NOT deserialize the payload so it is fast
-          #
-          # @note We assume that there can be different number of partitions in the target topic,
-          #   this is why we use `key` based on the source topic key and not the partition id.
-          #   This will not utilize partitions beyond the number of partitions of source topic,
-          #   but will accommodate for topics with less partitions.
-          def pipe_async(topic:, message:)
-            produce_async(
-              build_pipe_message(topic: topic, message: message)
-            )
-          end
-
-          # Sync version of pipe for one message
-          #
-          # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] source message to pipe
-          # @see [#pipe_async]
-          def pipe_sync(topic:, message:)
-            produce_sync(
-              build_pipe_message(topic: topic, message: message)
-            )
-          end
-
-          # Async multi-message pipe
-          #
-          # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
-          #
-          # @note If transactional producer in use and dispatch is not wrapped with a transaction,
-          #   it will automatically wrap the dispatch with a transaction
-          def pipe_many_async(topic:, messages:)
-            messages = messages.map do |message|
-              build_pipe_message(topic: topic, message: message)
-            end
-
-            produce_many_async(messages)
-          end
-
-          # Sync multi-message pipe
-          #
-          # @param topic [String, Symbol] where we want to send the message
-          # @param messages [Array<Karafka::Messages::Message>] source messages to pipe
-          #
-          # @note If transactional producer in use and dispatch is not wrapped with a transaction,
-          #   it will automatically wrap the dispatch with a transaction
-          def pipe_many_sync(topic:, messages:)
-            messages = messages.map do |message|
-              build_pipe_message(topic: topic, message: message)
-            end
-
-            produce_many_sync(messages)
-          end
-
-          private
-
-          # @param topic [String, Symbol] where we want to send the message
-          # @param message [Karafka::Messages::Message] source message to pipe
-          # @return [Hash] hash with message to pipe.
-          #
-          # @note If you need to alter this, please define the `#enhance_pipe_message` method
-          def build_pipe_message(topic:, message:)
-            pipe_message = {
-              topic: topic,
-              payload: message.raw_payload,
-              headers: message.raw_headers.merge(
-                "source_topic" => message.topic,
-                "source_partition" => message.partition.to_s,
-                "source_offset" => message.offset.to_s,
-                "source_consumer_group" => self.topic.consumer_group.id
-              )
-            }
-
-            # Use a key only if key was provided
-            pipe_message[:key] = if message.raw_key
-              message.raw_key
-            # Otherwise pipe creating a key that will assign it based on the source partition
-            # number
-            else
-              message.partition.to_s
-            end
-
-            # Optional method user can define in consumer to enhance the dlq message hash with
-            # some extra details if needed or to replace payload, etc
-            if respond_to?(:enhance_pipe_message, true)
-              enhance_pipe_message(
-                pipe_message,
-                message
-              )
-            end
-
-            pipe_message
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # DLQ enabled
-          # Filtering enabled
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          # Virtual Partitions enabled
-          module DlqFtrLrjMomVp
-            include Strategies::Aj::FtrMom
-            include Strategies::Aj::DlqMomVp
-            include Strategies::Aj::LrjMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              filtering
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            # This strategy assumes we do not early break on shutdown as it has VP
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Since we have VP here we do not commit intermediate offsets and need to commit
-                  # them here. We do commit in collapsed mode but this is generalized.
-                  mark_as_consumed(last_group_message) unless revoked?
-
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked?
-                    # no need to check for manual seek because AJ consumer is internal and
-                    # fully controlled by us
-                    seek(seek_offset, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # - Aj
-          # - Dlq
-          # - Ftr
-          # - Mom
-          module DlqFtrMom
-            include Strategies::Dlq::FtrMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              filtering
-              manual_offset_management
-            ].freeze
-
-            # We write our own custom handler for after consume here, because we mark as consumed
-            # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
-            # dispatch, we can mark this message as consumed as well.
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  handle_post_filtering
-                # If we've reached number of retries that we could, we need to skip the first
-                # message that was not marked as consumed, pause and continue, while also moving
-                # this message to the dead topic.
-                #
-                # For a Mom setup, this means, that user has to manage the checkpointing by
-                # himself. If no checkpointing is ever done, we end up with an endless loop.
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    # We can commit the offset here because we know that we skip it "forever" and
-                    # since AJ consumer commits the offset after each job, we also know that the
-                    # previous job was successful
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # - Aj
-          # - Dlq
-          # - Ftr
-          # - Mom
-          # - VP
-          module DlqFtrMomVp
-            include Strategies::Aj::DlqMomVp
-            include Strategies::Aj::DlqFtrMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              filtering
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
-            # end of all VPs
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  mark_as_consumed(last_group_message)
-
-                  handle_post_filtering
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    # We can commit the offset here because we know that we skip it "forever" and
-                    # since AJ consumer commits the offset after each job, we also know that the
-                    # previous job was successful
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # DLQ enabled
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          #
-          # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
-          # offset management works in this case.
-          module DlqLrjMom
-            include Strategies::Default
-            include Strategies::Dlq::Default
-            include Strategies::Aj::LrjMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              long_running_job
-              manual_offset_management
-            ].freeze
-
-            # We cannot use a VP version of this, because non-VP can early stop on shutdown
-            def handle_after_consume
-              coordinator.on_finished do
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # no need to check for manual seek because AJ consumer is internal and
-                  # fully controlled by us
-                  seek(seek_offset, false, reset_offset: false) unless revoked?
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # DLQ enabled
-          # Long-Running Job enabled
-          # Manual offset management enabled
-          # Virtual Partitions enabled
-          #
-          # This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
-          # offset management works in this case.
-          module DlqLrjMomVp
-            include Strategies::Aj::DlqMomVp
-            include Strategies::Aj::LrjMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-
-            # This strategy assumes we do not early break on shutdown as it has VP
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Since we have VP here we do not commit intermediate offsets and need to commit
-                  # them here. We do commit in collapsed mode but this is generalized.
-                  mark_as_consumed(last_group_message) unless revoked?
-                  # no need to check for manual seek because AJ consumer is internal and
-                  # fully controlled by us
-                  seek(seek_offset, false, reset_offset: false) unless revoked?
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Aj
-          # ActiveJob enabled
-          # DLQ enabled
-          # Manual offset management enabled
-          #
-          # AJ has manual offset management on by default and the offset management is delegated to
-          # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
-          # consumed when we skip given job upon errors. In all the other scenarios marking as
-          # consumed needs to happen in the AJ consumer on a per job basis.
-          module DlqMom
-            include Strategies::Dlq::Mom
-
-            # Features for this strategy
-            FEATURES = %i[
-              active_job
-              dead_letter_queue
-              manual_offset_management
-            ].freeze
-
-            # How should we post-finalize consumption.
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
-                  coordinator.pause_tracker.reset
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                    # We can commit the offset here because we know that we skip it "forever" and
-                    # since AJ consumer commits the offset after each job, we also know that the
-                    # previous job was successful
-                    mark_dispatched_to_dlq(skippable_message)
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end