karafka 2.0.38 → 2.0.40

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the strategies starting with DLQ
+        module Dlq
+          # Only dead letter queue enabled
+          module Default
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+            ].freeze
+
+            # When we encounter non-recoverable message, we skip it and go on with our lives
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(messages.last)
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+
+            # Finds the message may want to skip (all, starting from first)
+            # @private
+            # @return [Array<Karafka::Messages::Message, Boolean>] message we may want to skip and
+            #   information if this message was from marked offset or figured out via mom flow
+            def find_skippable_message
+              skippable_message = messages.find do |msg|
+                coordinator.marked? && msg.offset == coordinator.seek_offset
+              end
+
+              # If we don't have the message matching the last comitted offset, it means that
+              # user operates with manual offsets and we're beyond the batch in which things
+              # broke for the first time. Then we skip the first (as no markings) and we
+              # move on one by one.
+              skippable_message ? [skippable_message, true] : [messages.first, false]
+            end
+
+            # Moves the broken message into a separate queue defined via the settings
+            #
+            # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
+            #   dispatch to DLQ
+            def dispatch_to_dlq(skippable_message)
+              producer.produce_async(
+                build_dlq_message(
+                  skippable_message
+                )
+              )
+
+              # Notify about dispatch on the events bus
+              Karafka.monitor.instrument(
+                'dead_letter_queue.dispatched',
+                caller: self,
+                message: skippable_message
+              )
+            end
+
+            # @param skippable_message [Array<Karafka::Messages::Message>]
+            # @return [Hash] dispatch DLQ message
+            def build_dlq_message(skippable_message)
+              original_partition = skippable_message.partition.to_s
+
+              dlq_message = {
+                topic: topic.dead_letter_queue.topic,
+                key: original_partition,
+                payload: skippable_message.raw_payload,
+                headers: skippable_message.headers.merge(
+                  'original_topic' => topic.name,
+                  'original_partition' => original_partition,
+                  'original_offset' => skippable_message.offset.to_s,
+                  'original_consumer_group' => topic.consumer_group.id
+                )
+              }
+
+              # Optional method user can define in consumer to enhance the dlq message hash with
+              # some extra details if needed or to replace payload, etc
+              if respond_to?(:enhance_dlq_message, true)
+                enhance_dlq_message(
+                  dlq_message,
+                  skippable_message
+                )
+              end
+
+              dlq_message
+            end
+
+            # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch
+            #   topic is set to false, we will skip the dispatch, effectively ignoring the broken
+            #   message without taking any action.
+            def dispatch_to_dlq?
+              topic.dead_letter_queue.topic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::Vp
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+            ].freeze
+
+            # DLQ flow is standard here, what is not, is the success component where we need to
+            # take into consideration the filtering
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(messages.last)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          module FtrLrj
+            include Strategies::Dlq::Lrj
+            include Strategies::Lrj::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+            ].freeze
+
+            # This is one of more complex cases.
+            # We need to ensure, that we always resume (inline or via paused backoff) and we need
+            # to make sure we dispatch to DLQ when needed. Because revocation on LRJ can happen
+            # any time, we need to make sure we do not dispatch to DLQ when error happens but we
+            # no longer own the assignment. Throttling is another factor that has to be taken
+            # into consideration on the successful path
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          module FtrLrjMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # Post execution flow of this strategy
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(last_group_message.offset + 1)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  if marked
+                    pause(coordinator.seek_offset, nil, false)
+                  else
+                    pause(skippable_message.offset + 1, nil, false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/{mom.rb → dlq/ftr_lrj_vp.rb}

@@ -15,26 +15,23 @@ module Karafka
   module Pro
     module Processing
       module Strategies
-        # Manual offset management enabled
-        module Mom
-          include Default
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Virtual Partitions
+          module FtrLrjVp
+            include Strategies::Vp::Default
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrj
 
-          # Features for this strategy
-          FEATURES = %i[
-            manual_offset_management
-          ].freeze
-
-          # When mom is enabled, we do not mark messages as consumed after processing
-          def handle_after_consume
-            coordinator.on_finished do
-              return if revoked?
-
-              if coordinator.success?
-                coordinator.pause_tracker.reset
-              else
-                retry_after_pause
-              end
-            end
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              virtual_partitions
+            ].freeze
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          module FtrMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # On mom we do not mark, throttling and seeking as in other strategies
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+
+                  skippable_message, marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  # Move beyond the broken message
+                  if marked
+                    pause(coordinator.seek_offset, nil, false)
+                  else
+                    pause(skippable_message.offset + 1, nil, false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - VPs
+          #
+          # Behaves same as non-VP due to coordinator lock
+          module FtrVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Long-Running Job enabled
+          module Lrj
+            # Order here matters, lrj needs to be second
+            include Strategies::Dlq::Default
+            include Strategies::Lrj::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+            ].freeze
+
+            # LRJ standard flow after consumption with DLQ dispatch
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message) unless revoked?
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom and LRJ enabled
+          module LrjMom
+            # This strategy needs to pause and revoke same way as DlqLrj but without the offset
+            # management
+            include Strategies::Dlq::Lrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # LRJ standard flow after consumption with DLQ dispatch and no offset management
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  seek(last_group_message.offset + 1) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  if marked
+                    pause(coordinator.seek_offset, nil, false)
+                  else
+                    pause(skippable_message.offset + 1, nil, false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Long-Running Job enabled
+          # Virtual Partitions enabled
+          module LrjVp
+            # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Lrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end