karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Dead-Letter Queue enabled
-          # Filtering enabled
-          # Long-Running Job enabled
-          module FtrLrj
-            include Strategies::Dlq::Lrj
-            include Strategies::Lrj::Ftr
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              long_running_job
-            ].freeze
-
-            # This is one of more complex cases.
-            # We need to ensure, that we always resume (inline or via paused backoff) and we need
-            # to make sure we dispatch to DLQ when needed. Because revocation on LRJ can happen
-            # any time, we need to make sure we do not dispatch to DLQ when error happens but we
-            # no longer own the assignment. Throttling is another factor that has to be taken
-            # into consideration on the successful path
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  mark_as_consumed(last_group_message) unless revoked?
-
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked? && !coordinator.manual_seek?
-                    seek(seek_offset, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    return resume if revoked?
-
-                    dispatch_if_needed_and_mark_as_consumed
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # DLQ enabled
-          # Ftr enabled
-          # LRJ enabled
-          # MoM enabled
-          module FtrLrjMom
-            include Strategies::Ftr::Default
-            include Strategies::Dlq::LrjMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              long_running_job
-              manual_offset_management
-            ].freeze
-
-            # Post execution flow of this strategy
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked? && !coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    return resume if revoked?
-
-                    skippable_message, _marked = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-
-                    if mark_after_dispatch?
-                      mark_dispatched_to_dlq(skippable_message)
-                    else
-                      self.seek_offset = skippable_message.offset + 1
-                    end
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # DLQ enabled
-          # Ftr enabled
-          # LRJ enabled
-          # MoM enabled
-          # VP enabled
-          module FtrLrjMomVp
-            include Strategies::Vp::Default
-            include Strategies::Dlq::Vp
-            # Same as non VP because of the coordinator post-execution lock
-            include Strategies::Dlq::FtrLrjMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              long_running_job
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Dead-Letter Queue enabled
-          # Filtering enabled
-          # Long-Running Job enabled
-          # Virtual Partitions
-          module FtrLrjVp
-            include Strategies::Vp::Default
-            include Strategies::Dlq::Vp
-            # Same as non VP because of the coordinator post-execution lock
-            include Strategies::Dlq::FtrLrj
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              long_running_job
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # - DLQ
-          # - Ftr
-          # - Mom
-          module FtrMom
-            include Strategies::Ftr::Default
-            include Strategies::Dlq::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              manual_offset_management
-            ].freeze
-
-            # On mom we do not mark, throttling and seeking as in other strategies
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  handle_post_filtering
-                else
-                  apply_dlq_flow do
-                    skippable_message, _marked = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-
-                    if mark_after_dispatch?
-                      mark_dispatched_to_dlq(skippable_message)
-                    else
-                      self.seek_offset = skippable_message.offset + 1
-                    end
-                  end
-                end
-              end
-            end
-
-            # @return [Boolean] should we mark given message as consumed after dispatch. For
-            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
-            #  is `nil`, which means `false` in this case. If user provided alternative value, we
-            #  go with it.
-            #
-            # @note Please note, this is the opposite behavior than in case of AOM strategies.
-            def mark_after_dispatch?
-              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-              topic.dead_letter_queue.mark_after_dispatch
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # - DLQ
-          # - Ftr
-          # - Mom
-          # - VP
-          # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
-          module FtrMomVp
-            include Strategies::Mom::Vp
-            include Strategies::Dlq::Vp
-            include Strategies::Dlq::FtrMom
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # - DLQ
-          # - Ftr
-          # - VPs
-          #
-          # Behaves same as non-VP due to coordinator lock
-          module FtrVp
-            include Strategies::Vp::Default
-            include Strategies::Dlq::Vp
-            include Strategies::Dlq::Ftr
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              filtering
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Strategy for supporting DLQ with Mom and LRJ enabled
-          module LrjMom
-            # This strategy needs to pause and revoke same way as DlqLrj but without the offset
-            # management
-            include Strategies::Dlq::Lrj
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              long_running_job
-              manual_offset_management
-            ].freeze
-
-            # LRJ standard flow after consumption with DLQ dispatch and no offset management
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  unless revoked? || coordinator.manual_seek?
-                    seek(last_group_message.offset + 1, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  apply_dlq_flow do
-                    return resume if revoked?
-
-                    skippable_message, _marked = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-
-                    if mark_after_dispatch?
-                      mark_dispatched_to_dlq(skippable_message)
-                    else
-                      self.seek_offset = skippable_message.offset + 1
-                    end
-                  end
-                end
-              end
-            end
-
-            # @return [Boolean] should we mark given message as consumed after dispatch. For
-            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
-            #  is `nil`, which means `false` in this case. If user provided alternative value, we
-            #  go with it.
-            #
-            # @note Please note, this is the opposite behavior than in case of AOM strategies.
-            def mark_after_dispatch?
-              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-              topic.dead_letter_queue.mark_after_dispatch
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Dead-Letter Queue enabled
-          # Long-Running Job enabled
-          # Virtual Partitions enabled
-          module LrjVp
-            # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
-            include Strategies::Vp::Default
-            include Strategies::Dlq::Vp
-            include Strategies::Dlq::Lrj
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              long_running_job
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end