karafka 2.0.37 → 2.0.39

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          module DlqFtrMom
+            include Strategies::Dlq::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # We write our own custom handler for after consume here, because we mark as consumed
+            # per each job in the AJ consumer itself (for this strategy). This means, that for DLQ
+            # dispatch, we can mark this message as consumed as well.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # - Aj
+          # - Dlq
+          # - Ftr
+          # - Mom
+          # - VP
+          module DlqFtrMomVp
+            include Strategies::Vp::Default
+            include Strategies::Aj::DlqFtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ VP does not early stop on shutdown, hence here we can mark as consumed at the
+            # end of all VPs
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          #
+          # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+          # offset management works in this case.
+          module DlqLrjMom
+            include Strategies::Aj::DlqLrjMomVp
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # We cannot use a VP version of this, because non-VP can early stop on shutdown
+            def handle_after_consume
+              coordinator.on_finished do
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          #
+          # This case is a bit of special. Please see the `Aj::DlqMom` for explanation on how the
+          # offset management works in this case.
+          module DlqLrjMomVp
+            include Strategies::Aj::DlqMomVp
+            include Strategies::Aj::LrjMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # This strategy assumes we do not early break on shutdown as it has VP
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Since we have VP here we do not commit intermediate offsets and need to commit
+                  # them here. We do commit in collapsed mode but this is generalized.
+                  mark_as_consumed(last_group_message) unless revoked?
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # DLQ enabled
+          # Manual offset management enabled
+          #
+          # AJ has manual offset management on by default and the offset management is delegated to
+          # the AJ consumer. This means, we cannot mark as consumed always. We can only mark as
+          # consumed when we skip given job upon errors. In all the other scenarios marking as
+          # consumed needs to happen in the AJ consumer on a per job basis.
+          module DlqMom
+            include Strategies::Dlq::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              manual_offset_management
+            ].freeze
+
+            # How should we post-finalize consumption.
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
+                  coordinator.pause_tracker.reset
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  # We can commit the offset here because we know that we skip it "forever" and
+                  # since AJ consumer commits the offset after each job, we also know that the
+                  # previous job was successful
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqMomVp
+            include Strategies::Dlq::Default
+            include Strategies::Vp::Default
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # Flow including moving to DLQ in the collapsed mode
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # When this is an ActiveJob running via Pro with virtual partitions, we cannot
+                  # mark intermediate jobs as processed not to mess up with the ordering.
+                  # Only when all the jobs are processed and we did not loose the partition
+                  # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                  # commit offsets .
+                  # For a non virtual partitions case, the flow is regular and state is marked
+                  # after each successfully processed job
+                  return if revoked?
+
+                  mark_as_consumed(last_group_message)
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  # Here we are in a collapsed state, hence we can apply the same logic as
+                  # Aj::DlqMom
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module FtrLrjMom
+            include Strategies::Lrj::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Lrj::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ MOM VP does not do intermediate marking, hence we need to make sure we mark as
+            # consumed here.
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for ActiveJob related strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Manual Offset management enabled
+          module FtrMom
+            # Same as standard Mom::Ftr
+            include Strategies::Mom::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for ActiveJob related strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Manual Offset management enabled
+          # Virtual partitions enabled
+          module FtrMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::MomVp
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ with VPs always has intermediate marking disabled, hence we need to do it post
+            # execution always.
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if revoked?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/{dlq_lrj_vp.rb → aj/lrj_mom.rb}

@@ -15,20 +15,21 @@ module Karafka
   module Pro
     module Processing
       module Strategies
-        # Dead-Letter Queue enabled
-        # Long-Running Job enabled
-        # Virtual Partitions enabled
-        module DlqLrjVp
-          # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
-          include Vp
-          include DlqLrj
+        module Aj
+          # ActiveJob enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module LrjMom
+            # Same behaviour as Lrj::Mom
+            include Strategies::Lrj::Mom
 
-          # Features for this strategy
-          FEATURES = %i[
-            dead_letter_queue
-            long_running_job
-            virtual_partitions
-          ].freeze
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              long_running_job
+              manual_offset_management
+            ].freeze
+          end
         end
       end
     end