karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Strategy for supporting DLQ with Mom enabled
-          module Mom
-            # The broken message lookup is the same in this scenario
-            include Strategies::Dlq::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              manual_offset_management
-            ].freeze
-
-            # When manual offset management is on, we do not mark anything as consumed
-            # automatically and we rely on the user to figure things out
-            def handle_after_consume
-              coordinator.on_finished do
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-                else
-                  apply_dlq_flow do
-                    skippable_message, = find_skippable_message
-                    dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-
-                    if mark_after_dispatch?
-                      mark_dispatched_to_dlq(skippable_message)
-                    else
-                      # Save the next offset we want to go with after moving given message to DLQ
-                      # Without this, we would not be able to move forward and we would end up
-                      # in an infinite loop trying to un-pause from the message we've already
-                      # processed. Of course, since it's a MoM a rebalance or kill, will move it
-                      # back as no offsets are being committed
-                      self.seek_offset = skippable_message.offset + 1
-                    end
-                  end
-                end
-              end
-            end
-
-            # @return [Boolean] should we mark given message as consumed after dispatch. For
-            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
-            #  is `nil`, which means `false` in this case. If user provided alternative value, we
-            #  go with it.
-            #
-            # @note Please note, this is the opposite behavior than in case of AOM strategies.
-            def mark_after_dispatch?
-              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
-
-              topic.dead_letter_queue.mark_after_dispatch
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Dead-Letter Queue enabled
-          # MoM enabled
-          # Virtual Partitions enabled
-          module MomVp
-            include Strategies::Dlq::Vp
-            include Strategies::Dlq::Mom
-
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              manual_offset_management
-              virtual_partitions
-            ].freeze
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq/vp.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Dlq
-          # Dead Letter Queue enabled
-          # Virtual Partitions enabled
-          #
-          # In general because we collapse processing in virtual partitions to one on errors, there
-          # is no special action that needs to be taken because we warranty that even with VPs
-          # on errors a retry collapses into a single state and from this single state we can
-          # mark as consumed the message that we are moving to the DLQ.
-          module Vp
-            # Features for this strategy
-            FEATURES = %i[
-              dead_letter_queue
-              virtual_partitions
-            ].freeze
-
-            include Strategies::Dlq::Default
-            include Strategies::Vp::Default
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -1,141 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Filtering related init strategies
-        module Ftr
-          # Only filtering enabled
-          module Default
-            include Strategies::Default
-
-            # Just filtering enabled
-            FEATURES = %i[
-              filtering
-            ].freeze
-
-            # Empty run when running on idle means we need to filter
-            def handle_idle
-              handle_post_filtering
-            end
-
-            # Standard flow without any features
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                return if revoked?
-
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Do not mark last message if pause happened. This prevents a scenario where
-                  # pause is overridden upon rebalance by marking
-                  return if coordinator.manual_pause?
-
-                  mark_as_consumed(last_group_message)
-
-                  handle_post_filtering
-                else
-                  retry_after_pause
-                end
-              end
-            end
-
-            # Throttles by pausing for an expected time period if throttling is needed or seeks
-            # in case the throttle expired. Throttling may expire because we throttle before
-            # processing starts and we need to compensate for processing time. It may turn out
-            # that we don't have to pause but we need to move the offset because we skipped some
-            # messages due to throttling filtering.
-            # @return [Boolean] was any form of throttling operations (pause or seek) needed
-            def handle_post_filtering
-              filter = coordinator.filter
-
-              # We pick the timeout before the action because every action takes time. This time
-              # may then mean we end up having throttle time equal to zero when pause is needed
-              # and this should not happen
-              throttle_timeout = filter.timeout
-
-              # If user requested marking when applying filter, we mark. We may be in the user
-              # flow but even then this is not a problem. Older offsets will be ignored since
-              # we do not force the offset update (expected) and newer are on the user to control.
-              # This can be primarily used when filtering large quantities of data to mark on the
-              # idle runs, so lag reporting is aware that those messages were not consumed but also
-              # are no longer relevant
-              if filter.mark_as_consumed?
-                send(
-                  filter.marking_method,
-                  filter.marking_cursor
-                )
-              end
-
-              case filter.action
-              when :skip
-                nil
-              when :seek
-                # User direct actions take priority over automatic operations
-                # If we've already seeked we can just resume operations, nothing extra needed
-                return resume if coordinator.manual_seek?
-
-                throttle_message = filter.cursor
-
-                monitor.instrument(
-                  "filtering.seek",
-                  caller: self,
-                  message: throttle_message
-                ) do
-                  seek(throttle_message.offset, false)
-                end
-
-                resume
-              when :pause
-                # User direct actions take priority over automatic operations
-                return nil if coordinator.manual_pause?
-
-                throttle_message = filter.cursor
-
-                monitor.instrument(
-                  "filtering.throttled",
-                  caller: self,
-                  message: throttle_message,
-                  timeout: throttle_timeout
-                ) do
-                  pause(throttle_message.offset, throttle_timeout, false)
-                end
-              else
-                raise Karafka::Errors::UnsupportedCaseError filter.action
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/ftr/vp.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Filtering related init strategies
-        module Ftr
-          # Filtering enabled
-          # VPs enabled
-          #
-          # VPs should operate without any problems with filtering because virtual partitioning
-          # happens on the limited set of messages and collective filtering applies the same
-          # way as for default cases
-          module Vp
-            # Filtering + VPs
-            FEATURES = %i[
-              filtering
-              virtual_partitions
-            ].freeze
-
-            include Strategies::Vp::Default
-            include Strategies::Ftr::Default
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -1,114 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Lrj
-          # Long-Running Job enabled
-          module Default
-            include Strategies::Default
-
-            # Pause for tops 31 years
-            MAX_PAUSE_TIME = 1_000_000_000_000
-
-            # Features for this strategy
-            FEATURES = %i[
-              long_running_job
-            ].freeze
-
-            # We always need to pause prior to doing any jobs for LRJ
-            def handle_before_schedule_consume
-              super
-
-              # This ensures that when running LRJ with VP, things operate as expected run only
-              # once for all the virtual partitions collectively
-              coordinator.on_enqueued do
-                # Pause and continue with another batch in case of a regular resume.
-                # In case of an error, the `#retry_after_pause` will move the offset to the first
-                # message out of this batch.
-                pause(:consecutive, MAX_PAUSE_TIME, false)
-              end
-            end
-
-            # LRJ standard flow after consumption
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  return if coordinator.manual_pause?
-
-                  mark_as_consumed(last_group_message) unless revoked?
-
-                  unless revoked? || coordinator.manual_seek?
-                    seek(seek_offset, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  # If processing failed, we need to pause
-                  # For long running job this will overwrite the default never-ending pause and
-                  # will cause the processing to keep going after the error backoff
-                  retry_after_pause
-                end
-              end
-            end
-
-            # We do not un-pause on revokations for LRJ
-            def handle_revoked
-              coordinator.on_revoked do
-                # We do not want to resume on revocation in case of a LRJ.
-                # For LRJ we resume after the successful processing or do a backoff pause in case
-                # of a failure. Double non-blocking resume could cause problems in coordination.
-                coordinator.revoke
-              end
-
-              monitor.instrument("consumer.revoke", caller: self)
-              monitor.instrument("consumer.revoked", caller: self) do
-                revoked
-              end
-            ensure
-              coordinator.decrement(:revoked)
-            end
-
-            # Allows for LRJ to synchronize its work. It may be needed because LRJ can run
-            # lifecycle events like revocation while the LRJ work is running and there may be a
-            # need for a critical section.
-            def synchronize(&)
-              coordinator.shared_mutex.synchronize(&)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Lrj
-          # Filtering enabled
-          # Long-Running Job enabled
-          #
-          # In general aside from throttling this one will behave the same way as the Lrj
-          module Ftr
-            include Strategies::Ftr::Default
-            include Strategies::Lrj::Default
-
-            # Features for this strategy
-            FEATURES = %i[
-              filtering
-              long_running_job
-            ].freeze
-
-            # LRJ standard flow after consumption with potential throttling on success
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Manual pausing has the highest priority
-                  return if coordinator.manual_pause?
-
-                  # It's not a MoM, so for successful we need to mark as consumed
-                  mark_as_consumed(last_group_message) unless revoked?
-
-                  # If still not revoked and was throttled, we need to apply throttling logic
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked? && !coordinator.manual_seek?
-                    # If not revoked and not throttled, we move to where we were suppose to and
-                    # resume
-                    seek(seek_offset, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  # If processing failed, we need to pause
-                  # For long running job this will overwrite the default never-ending pause and
-                  # will cause the processing to keep going after the error backoff
-                  retry_after_pause
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        module Lrj
-          # Long-Running Job enabled
-          # Filtering enabled
-          # Manual offset management enabled
-          #
-          # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
-          module FtrMom
-            include Strategies::Lrj::Ftr
-
-            # Features for this strategy
-            FEATURES = %i[
-              filtering
-              long_running_job
-              manual_offset_management
-            ].freeze
-
-            # LRJ standard flow after consumption with potential filtering on success
-            def handle_after_consume
-              coordinator.on_finished do |last_group_message|
-                if coordinator.success?
-                  coordinator.pause_tracker.reset
-
-                  # Manual pausing has the highest priority
-                  return if coordinator.manual_pause?
-
-                  # If still not revoked and was throttled, we need to apply filtering logic
-                  if coordinator.filtered? && !revoked?
-                    handle_post_filtering
-
-                    # :seek and :pause are fully handled by handle_post_filtering
-                    # For :skip we still need to resume the LRJ MAX_PAUSE_TIME pause
-                    return unless coordinator.filter.action == :skip
-                  elsif !revoked? && !coordinator.manual_seek?
-                    # If not revoked and not throttled, we move to where we were suppose to and
-                    # resume
-                    seek(last_group_message.offset + 1, false, reset_offset: false)
-                  end
-
-                  resume
-                else
-                  # If processing failed, we need to pause
-                  # For long running job this will overwrite the default never-ending pause and
-                  # will cause the processing to keep going after the error backoff
-                  retry_after_pause
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end