karafka 1.4.4 → 2.1.10

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          module FtrLrjMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # Post execution flow of this strategy
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked? && !coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                    resume
+                  else
+                    resume
+                  end
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, _marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Ftr enabled
+          # LRJ enabled
+          # MoM enabled
+          # VP enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Virtual Partitions
+          module FtrLrjVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            # Same as non VP because of the coordinator post-execution lock
+            include Strategies::Dlq::FtrLrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          module FtrMom
+            include Strategies::Ftr::Default
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # On mom we do not mark, throttling and seeking as in other strategies
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+
+                  skippable_message, _marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - Mom
+          # - VP
+          # We can use the `Strategies::FtrMom` because VPs collapse prior to DLQ
+          module FtrMomVp
+            include Strategies::Mom::Vp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # - DLQ
+          # - Ftr
+          # - VPs
+          #
+          # Behaves same as non-VP due to coordinator lock
+          module FtrVp
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              filtering
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # DLQ enabled
+          # Long-Running Job enabled
+          module Lrj
+            # Order here matters, lrj needs to be second
+            include Strategies::Dlq::Default
+            include Strategies::Lrj::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+            ].freeze
+
+            # LRJ standard flow after consumption with DLQ dispatch
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message) unless revoked?
+                  # We should not overwrite user manual seel request with our seek
+                  seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom and LRJ enabled
+          module LrjMom
+            # This strategy needs to pause and revoke same way as DlqLrj but without the offset
+            # management
+            include Strategies::Dlq::Lrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # LRJ standard flow after consumption with DLQ dispatch and no offset management
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  unless revoked? || coordinator.manual_seek?
+                    seek(last_group_message.offset + 1, false)
+                  end
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+
+                  return resume if revoked?
+
+                  skippable_message, _marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom_vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom, LRJ and VP enabled
+          module LrjMomVp
+            # Feature set for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::LrjMom
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/lrj_vp.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # Long-Running Job enabled
+          # Virtual Partitions enabled
+          module LrjVp
+            # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
+            include Strategies::Vp::Default
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Lrj
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom enabled
+          module Mom
+            # The broken message lookup is the same in this scenario
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              manual_offset_management
+            ].freeze
+
+            # When manual offset management is on, we do not mark anything as consumed
+            # automatically and we rely on the user to figure things out
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  # Save the next offset we want to go with after moving given message to DLQ
+                  # Without this, we would not be able to move forward and we would end up
+                  # in an infinite loop trying to un-pause from the message we've already processed
+                  # Of course, since it's a MoM a rebalance or kill, will move it back as no
+                  # offsets are being committed
+                  coordinator.seek_offset = skippable_message.offset + 1
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/mom_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead-Letter Queue enabled
+          # MoM enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Dlq::Vp
+            include Strategies::Dlq::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead Letter Queue enabled
+          # Virtual Partitions enabled
+          #
+          # In general because we collapse processing in virtual partitions to one on errors, there
+          # is no special action that needs to be taken because we warranty that even with VPs
+          # on errors a retry collapses into a single state and from this single state we can
+          # mark as consumed the message that we are moving to the DLQ.
+          module Vp
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Dlq::Default
+            include Strategies::Vp::Default
+          end
+        end
+      end
+    end
+  end
+end