karafka 2.0.37 → 2.0.39

data/lib/karafka/pro/processing/strategies/aj_dlq_mom.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # DLQ enabled
-        # Manual offset management enabled
-        #
-        # AJ has manual offset management on by default and the offset management is delegated to
-        # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
-        # consumed when we skip given job upon errors. In all the other scenarios marking as
-        # consumed needs to happen in the AJ consumer on a per job basis.
-        module AjDlqMom
-          include DlqMom
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            dead_letter_queue
-            manual_offset_management
-          ].freeze
-
-          # How should we post-finalize consumption.
-          def handle_after_consume
-            coordinator.on_finished do
-              return if revoked?
-
-              if coordinator.success?
-                # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
-                coordinator.pause_tracker.reset
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              else
-                coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                # We can commit the offset here because we know that we skip it "forever" and
-                # since AJ consumer commits the offset after each job, we also know that the
-                # previous job was successful
-                mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset, nil, false)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # Manual offset management enabled
-        # Virtual Partitions enabled
-        module AjDlqMomVp
-          include Dlq
-          include Vp
-          include Default
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            dead_letter_queue
-            manual_offset_management
-            virtual_partitions
-          ].freeze
-
-          # Flow including moving to DLQ in the collapsed mode
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
-                # intermediate jobs as processed not to mess up with the ordering.
-                # Only when all the jobs are processed and we did not loose the partition
-                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
-                # commit offsets on this as only then we can be sure, that all the jobs were
-                # processed.
-                # For a non virtual partitions case, the flow is regular and state is marked after
-                # each successfully processed job
-                return if revoked?
-                return if Karafka::App.stopping?
-
-                mark_as_consumed(last_group_message)
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              else
-                # Here we are in a collapsed state, hence we can apply the same logic as AjDlqMom
-                coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset, nil, false)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -1,75 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # Long-Running Job enabled
-        # Manual offset management enabled
-        # Virtual Partitions enabled
-        module AjLrjMomVp
-          include Default
-          include Vp
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            long_running_job
-            manual_offset_management
-            virtual_partitions
-          ].freeze
-
-          # No actions needed for the standard flow here
-          def handle_before_enqueue
-            coordinator.on_enqueued do
-              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME, false)
-            end
-          end
-
-          # Standard flow without any features
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                mark_as_consumed(last_group_message) unless revoked? || Karafka::App.stopping?
-                seek(coordinator.seek_offset) unless revoked?
-
-                resume
-              else
-                # If processing failed, we need to pause
-                # For long running job this will overwrite the default never-ending pause and will
-                # cause the processing to keep going after the error backoff
-                retry_after_pause
-              end
-            end
-          end
-
-          # LRJ cannot resume here. Only in handling the after consumption
-          def handle_revoked
-            coordinator.on_revoked do
-              coordinator.revoke
-            end
-
-            Karafka.monitor.instrument('consumer.revoke', caller: self)
-            Karafka.monitor.instrument('consumer.revoked', caller: self) do
-              revoked
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # Manual offset management enabled
-        # Virtual Partitions enabled
-        module AjMomVp
-          include Vp
-          include Default
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            manual_offset_management
-            virtual_partitions
-          ].freeze
-
-          # Standard flow without any features
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
-                # intermediate jobs as processed not to mess up with the ordering.
-                # Only when all the jobs are processed and we did not loose the partition
-                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
-                # commit offsets on this as only then we can be sure, that all the jobs were
-                # processed.
-                # For a non virtual partitions case, the flow is regular and state is marked after
-                # each successfully processed job
-                #
-                # We can mark and we do mark intermediate jobs in the collapsed mode when running
-                # VPs
-                return if revoked?
-                return if Karafka::App.stopping?
-
-                mark_as_consumed(last_group_message)
-              else
-                retry_after_pause
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -1,120 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Only dead letter queue enabled
-        module Dlq
-          include Default
-
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-          ].freeze
-
-          # When we encounter non-recoverable message, we skip it and go on with our lives
-          def handle_after_consume
-            coordinator.on_finished do
-              return if revoked?
-
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                return if coordinator.manual_pause?
-
-                mark_as_consumed(messages.last)
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              # If we've reached number of retries that we could, we need to skip the first message
-              # that was not marked as consumed, pause and continue, while also moving this message
-              # to the dead topic
-              else
-                # We reset the pause to indicate we will now consider it as "ok".
-                coordinator.pause_tracker.reset
-                skippable_message = find_skippable_message
-                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_as_consumed(skippable_message)
-                pause(coordinator.seek_offset, nil, false)
-              end
-            end
-          end
-
-          # Finds the message may want to skip (all, starting from first)
-          # @private
-          # @return [Karafka::Messages::Message] message we may want to skip
-          def find_skippable_message
-            skippable_message = messages.find { |msg| msg.offset == coordinator.seek_offset }
-            skippable_message || raise(Errors::SkipMessageNotFoundError, topic.name)
-          end
-
-          # Moves the broken message into a separate queue defined via the settings
-          #
-          # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
-          #   dispatch to DLQ
-          def dispatch_to_dlq(skippable_message)
-            producer.produce_async(
-              build_dlq_message(
-                skippable_message
-              )
-            )
-
-            # Notify about dispatch on the events bus
-            Karafka.monitor.instrument(
-              'dead_letter_queue.dispatched',
-              caller: self,
-              message: skippable_message
-            )
-          end
-
-          # @param skippable_message [Array<Karafka::Messages::Message>]
-          # @return [Hash] dispatch DLQ message
-          def build_dlq_message(skippable_message)
-            original_partition = skippable_message.partition.to_s
-
-            dlq_message = {
-              topic: topic.dead_letter_queue.topic,
-              key: original_partition,
-              payload: skippable_message.raw_payload,
-              headers: skippable_message.headers.merge(
-                'original_topic' => topic.name,
-                'original_partition' => original_partition,
-                'original_offset' => skippable_message.offset.to_s,
-                'original_consumer_group' => topic.consumer_group.id
-              )
-            }
-
-            # Optional method user can define in consumer to enhance the dlq message hash with
-            # some extra details if needed or to replace payload, etc
-            if respond_to?(:enhance_dlq_message, true)
-              enhance_dlq_message(
-                dlq_message,
-                skippable_message
-              )
-            end
-
-            dlq_message
-          end
-
-          # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
-          #   is set to false, we will skip the dispatch, effectively ignoring the broken message
-          #   without taking any action.
-          def dispatch_to_dlq?
-            topic.dead_letter_queue.topic
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq_lrj.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # DLQ enabled
-        # Long-Running Job enabled
-        module DlqLrj
-          # Order here matters, lrj needs to be second
-          include Dlq
-          include Lrj
-
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-            long_running_job
-          ].freeze
-
-          # LRJ standard flow after consumption with DLQ dispatch
-          def handle_after_consume
-            coordinator.on_finished do |last_group_message|
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                return if coordinator.manual_pause?
-
-                mark_as_consumed(last_group_message) unless revoked?
-                seek(coordinator.seek_offset) unless revoked?
-
-                resume
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              else
-                coordinator.pause_tracker.reset
-
-                unless revoked?
-                  skippable_message = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
-                end
-
-                # This revoke might have changed state due to marking, hence checked again
-                seek(coordinator.seek_offset) unless revoked?
-
-                resume
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Strategy for supporting DLQ with Mom and LRJ enabled
-        module DlqLrjMom
-          # This strategy needs to pause and revoke same way as DlqLrj but without the offset
-          # management
-          include DlqLrj
-
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-            long_running_job
-            manual_offset_management
-          ].freeze
-
-          # LRJ standard flow after consumption with DLQ dispatch and no offset management
-          def handle_after_consume
-            coordinator.on_finished do
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-
-                seek(coordinator.seek_offset) unless revoked?
-
-                resume
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              else
-                coordinator.pause_tracker.reset
-
-                unless revoked?
-                  if dispatch_to_dlq?
-                    skippable_message = find_skippable_message
-                    dispatch_to_dlq(skippable_message)
-                  end
-
-                  seek(coordinator.seek_offset)
-                end
-
-                resume
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Strategy for supporting DLQ with Mom enabled
-        module DlqMom
-          # The broken message lookup is the same in this scenario
-          include Dlq
-
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-            manual_offset_management
-          ].freeze
-
-          # When manual offset management is on, we do not mark anything as consumed automatically
-          # and we rely on the user to figure things out
-          def handle_after_consume
-            coordinator.on_finished do
-              return if revoked?
-
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
-                retry_after_pause
-              # If we've reached number of retries that we could, we need to skip the first message
-              # that was not marked as consumed, pause and continue, while also moving this message
-              # to the dead topic.
-              #
-              # For a Mom setup, this means, that user has to manage the checkpointing by himself.
-              # If no checkpointing is ever done, we end up with an endless loop.
-              else
-                # We reset the pause to indicate we will now consider it as "ok".
-                coordinator.pause_tracker.reset
-
-                if dispatch_to_dlq?
-                  skippable_message = find_skippable_message
-                  dispatch_to_dlq(skippable_message)
-                end
-
-                pause(coordinator.seek_offset, nil, false)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/strategies/dlq_vp.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
-
-module Karafka
-  module Pro
-    module Processing
-      module Strategies
-        # Dead Letter Queue enabled
-        # Virtual Partitions enabled
-        #
-        # In general because we collapse processing in virtual partitions to one on errors, there
-        # is no special action that needs to be taken because we warranty that even with VPs
-        # on errors a retry collapses into a single state.
-        module DlqVp
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-            virtual_partitions
-          ].freeze
-
-          include Dlq
-          include Vp
-        end
-      end
-    end
-  end
-end