karafka 2.0.38 → 2.0.40

data/lib/karafka/pro/processing/strategies/dlq/mom.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Strategy for supporting DLQ with Mom enabled
+          module Mom
+            # The broken message lookup is the same in this scenario
+            include Strategies::Dlq::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              manual_offset_management
+            ].freeze
+
+            # When manual offset management is on, we do not mark anything as consumed
+            # automatically and we rely on the user to figure things out
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                # If we've reached number of retries that we could, we need to skip the first
+                # message that was not marked as consumed, pause and continue, while also moving
+                # this message to the dead topic.
+                #
+                # For a Mom setup, this means, that user has to manage the checkpointing by
+                # himself. If no checkpointing is ever done, we end up with an endless loop.
+                else
+                  # We reset the pause to indicate we will now consider it as "ok".
+                  coordinator.pause_tracker.reset
+
+                  skippable_message, marked = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+
+                  # Move beyond the broken message
+                  if marked
+                    pause(coordinator.seek_offset, nil, false)
+                  else
+                    pause(skippable_message.offset + 1, nil, false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq/vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Dlq
+          # Dead Letter Queue enabled
+          # Virtual Partitions enabled
+          #
+          # In general because we collapse processing in virtual partitions to one on errors, there
+          # is no special action that needs to be taken because we warranty that even with VPs
+          # on errors a retry collapses into a single state and from this single state we can
+          # mark as consumed the message that we are moving to the DLQ.
+          module Vp
+            # Features for this strategy
+            FEATURES = %i[
+              dead_letter_queue
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Dlq::Default
+            include Strategies::Vp::Default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/ftr/default.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Filtering related init strategies
+        module Ftr
+          # Only filtering enabled
+          module Default
+            include Strategies::Default
+
+            # Just filtering enabled
+            FEATURES = %i[
+              filtering
+            ].freeze
+
+            # Empty run when running on idle means we need to filter
+            def handle_idle
+              handle_post_filtering
+            end
+
+            # Standard flow without any features
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Do not mark last message if pause happened. This prevents a scenario where
+                  # pause is overridden upon rebalance by marking
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                else
+                  retry_after_pause
+                end
+              end
+            end
+
+            # Throttles by pausing for an expected time period if throttling is needed or seeks
+            # in case the throttle expired. Throttling may expire because we throttle before
+            # processing starts and we need to compensate for processing time. It may turn out
+            # that we don't have to pause but we need to move the offset because we skipped some
+            # messages due to throttling filtering.
+            # @return [Boolean] was any form of throttling operations (pause or seek) needed
+            def handle_post_filtering
+              filter = coordinator.filter
+
+              # We pick the timeout before the action because every action takes time. This time
+              # may then mean we end up having throttle time equal to zero when pause is needed
+              # and this should not happen
+              throttle_timeout = filter.timeout
+
+              case filter.action
+              when :skip
+                nil
+              when :seek
+                throttle_message = filter.cursor
+
+                Karafka.monitor.instrument(
+                  'filtering.seek',
+                  caller: self,
+                  message: throttle_message
+                ) do
+                  seek(throttle_message.offset)
+                end
+
+                resume
+              when :pause
+                throttle_message = filter.cursor
+
+                Karafka.monitor.instrument(
+                  'filtering.throttled',
+                  caller: self,
+                  message: throttle_message,
+                  timeout: throttle_timeout
+                ) do
+                  pause(throttle_message.offset, throttle_timeout, false)
+                end
+              else
+                raise Karafka::Errors::UnsupportedCaseError filter.action
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/ftr/vp.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Filtering related init strategies
+        module Ftr
+          # Filtering enabled
+          # VPs enabled
+          #
+          # VPs should operate without any problems with filtering because virtual partitioning
+          # happens on the limited set of messages and collective filtering applies the same
+          # way as for default cases
+          module Vp
+            # Filtering + VPs
+            FEATURES = %i[
+              filtering
+              virtual_partitions
+            ].freeze
+
+            include Strategies::Vp::Default
+            include Strategies::Ftr::Default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/default.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Long-Running Job enabled
+          module Default
+            include Strategies::Default
+
+            # Pause for tops 31 years
+            MAX_PAUSE_TIME = 1_000_000_000_000
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+            ].freeze
+
+            # We always need to pause prior to doing any jobs for LRJ
+            def handle_before_enqueue
+              # This ensures that when running LRJ with VP, things operate as expected run only
+              # once for all the virtual partitions collectively
+              coordinator.on_enqueued do
+                # Pause at the first message in a batch. That way in case of a crash, we will not
+                # loose any messages.
+                #
+                # For VP it applies the same way and since VP cannot be used with MOM we should not
+                # have any edge cases here.
+                pause(coordinator.seek_offset, MAX_PAUSE_TIME, false)
+              end
+            end
+
+            # LRJ standard flow after consumption
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  mark_as_consumed(last_group_message) unless revoked?
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+
+            # We do not un-pause on revokations for LRJ
+            def handle_revoked
+              coordinator.on_revoked do
+                # We do not want to resume on revocation in case of a LRJ.
+                # For LRJ we resume after the successful processing or do a backoff pause in case
+                # of a failure. Double non-blocking resume could cause problems in coordination.
+                coordinator.revoke
+              end
+
+              Karafka.monitor.instrument('consumer.revoke', caller: self)
+              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+                revoked
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/ftr.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Filtering enabled
+          # Long-Running Job enabled
+          #
+          # In general aside from throttling this one will behave the same way as the Lrj
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Lrj::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+            ].freeze
+
+            # LRJ standard flow after consumption with potential throttling on success
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Manual pausing has the highest priority
+                  return if coordinator.manual_pause?
+
+                  # It's not a MoM, so for successful we need to mark as consumed
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  # If still not revoked and was throttled, we need to apply throttling logic
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    # If not revoked and not throttled, we move to where we were suppose to and
+                    # resume
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj/ftr_mom.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Filtering enabled
+          # Manual offset management enabled
+          #
+          # It is really similar to the Lrj::Ftr but we do not mark anything as consumed
+          module FtrMom
+            include Strategies::Lrj::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # LRJ standard flow after consumption with potential filtering on success
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Manual pausing has the highest priority
+                  return if coordinator.manual_pause?
+
+                  # If still not revoked and was throttled, we need to apply filtering logic
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    # If not revoked and not throttled, we move to where we were suppose to and
+                    # resume
+                    seek(last_group_message.offset + 1)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/{vp.rb → lrj/ftr_vp.rb}

@@ -15,20 +15,22 @@ module Karafka
   module Pro
     module Processing
       module Strategies
-        # Just Virtual Partitions enabled
-        module Vp
-          # This flow is exactly the same as the default one because the default one is wrapper
-          # with `coordinator#on_finished`
-          include Default
+        module Lrj
+          # Long-Running Job enabled
+          # Filtering enabled
+          # Virtual Partitions enabled
+          #
+          # Behaves same as non-VP because of the aggregated flow in the coordinator.
+          module FtrVp
+            include Strategies::Vp::Default
+            include Strategies::Lrj::Ftr
 
-          # Features for this strategy
-          FEATURES = %i[
-            virtual_partitions
-          ].freeze
-
-          # @return [Boolean] is the virtual processing collapsed in the context of given consumer.
-          def collapsed?
-            coordinator.collapsed?
+            # Features for this strategy
+            FEATURES = %i[
+              filtering
+              long_running_job
+              virtual_partitions
+            ].freeze
           end
         end
       end

data/lib/karafka/pro/processing/strategies/lrj/mom.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for all the LRJ starting strategies
+        module Lrj
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module Mom
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              manual_offset_management
+            ].freeze
+
+            # We always need to pause prior to doing any jobs for LRJ
+            def handle_before_enqueue
+              # This ensures that when running LRJ with VP, things operate as expected run only
+              # once for all the virtual partitions collectively
+              coordinator.on_enqueued do
+                # Pause at the first message in a batch. That way in case of a crash, we will not
+                # loose any messages.
+                #
+                # For VP it applies the same way and since VP cannot be used with MOM we should not
+                # have any edge cases here.
+                pause(coordinator.seek_offset, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
+              end
+            end
+
+            # No offset management, aside from that typical LRJ
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if coordinator.manual_pause?
+
+                  seek(last_group_message.offset + 1) unless revoked?
+
+                  resume
+                else
+                  retry_after_pause
+                end
+              end
+            end
+
+            # We do not un-pause on revokations for LRJ
+            def handle_revoked
+              coordinator.on_revoked do
+                coordinator.revoke
+              end
+
+              Karafka.monitor.instrument('consumer.revoke', caller: self)
+              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+                revoked
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/{aj_lrj_mom.rb → lrj/vp.rb}

@@ -15,19 +15,20 @@ module Karafka
   module Pro
     module Processing
       module Strategies
-        # ActiveJob enabled
-        # Long-Running Job enabled
-        # Manual offset management enabled
-        module AjLrjMom
-          # Same behaviour as LrjMom
-          include LrjMom
+        module Lrj
+          # Long-Running Job enabled
+          # Virtual Partitions enabled
+          module Vp
+            # Same flow as the standard Lrj
+            include Strategies::Vp::Default
+            include Strategies::Lrj::Default
 
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            long_running_job
-            manual_offset_management
-          ].freeze
+            # Features for this strategy
+            FEATURES = %i[
+              long_running_job
+              virtual_partitions
+            ].freeze
+          end
         end
       end
     end

data/lib/karafka/pro/processing/strategies/mom/default.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for Mom starting strategies
+        module Mom
+          # Manual offset management enabled
+          module Default
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              manual_offset_management
+            ].freeze
+
+            # When mom is enabled, we do not mark messages as consumed after processing
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end