karafka 2.0.0 → 2.1.0

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module DlqMomVp
+            include Strategies::Default
+            include Strategies::Dlq::Vp
+            include Strategies::Vp::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              dead_letter_queue
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # Flow including moving to DLQ in the collapsed mode
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # When this is an ActiveJob running via Pro with virtual partitions, we cannot
+                  # mark intermediate jobs as processed not to mess up with the ordering.
+                  # Only when all the jobs are processed and we did not loose the partition
+                  # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                  # commit offsets .
+                  # For a non virtual partitions case, the flow is regular and state is marked
+                  # after each successfully processed job
+                  return if revoked?
+
+                  mark_as_consumed(last_group_message)
+                elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                  retry_after_pause
+                else
+                  # Here we are in a collapsed state, hence we can apply the same logic as
+                  # Aj::DlqMom
+                  coordinator.pause_tracker.reset
+                  skippable_message, = find_skippable_message
+                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                  mark_as_consumed(skippable_message)
+                  pause(coordinator.seek_offset, nil, false)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module FtrLrjMom
+            include Strategies::Lrj::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              long_running_job
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module FtrLrjMomVp
+            include Strategies::Vp::Default
+            include Strategies::Lrj::FtrMom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ MOM VP does not do intermediate marking, hence we need to make sure we mark as
+            # consumed here.
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  mark_as_consumed(last_group_message) unless revoked?
+
+                  if coordinator.filtered? && !revoked?
+                    handle_post_filtering
+                  elsif !revoked?
+                    seek(coordinator.seek_offset)
+                    resume
+                  else
+                    resume
+                  end
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for ActiveJob related strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Manual Offset management enabled
+          module FtrMom
+            # Same as standard Mom::Ftr
+            include Strategies::Mom::Ftr
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for ActiveJob related strategies
+        module Aj
+          # ActiveJob enabled
+          # Filtering enabled
+          # Manual Offset management enabled
+          # Virtual partitions enabled
+          module FtrMomVp
+            include Strategies::Aj::FtrMom
+            include Strategies::Aj::MomVp
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              filtering
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # AJ with VPs always has intermediate marking disabled, hence we need to do it post
+            # execution always.
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if revoked?
+
+                  mark_as_consumed(last_group_message)
+
+                  handle_post_filtering
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          module LrjMom
+            # Same behaviour as Lrj::Mom
+            include Strategies::Lrj::Mom
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              long_running_job
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Long-Running Job enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module LrjMomVp
+            include Strategies::Default
+            include Strategies::Vp::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              long_running_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # No actions needed for the standard flow here
+            def handle_before_enqueue
+              super
+
+              coordinator.on_enqueued do
+                pause(coordinator.seek_offset, Strategies::Lrj::Default::MAX_PAUSE_TIME, false)
+              end
+            end
+
+            # Standard flow without any features
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  mark_as_consumed(last_group_message) unless revoked?
+                  seek(coordinator.seek_offset) unless revoked?
+
+                  resume
+                else
+                  # If processing failed, we need to pause
+                  # For long running job this will overwrite the default never-ending pause and
+                  # will cause the processing to keep going after the error backoff
+                  retry_after_pause
+                end
+              end
+            end
+
+            # LRJ cannot resume here. Only in handling the after consumption
+            def handle_revoked
+              coordinator.on_revoked do
+                coordinator.revoke
+              end
+
+              Karafka.monitor.instrument('consumer.revoke', caller: self)
+              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+                revoked
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/mom.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Namespace for ActiveJob related strategies
+        module Aj
+          # ActiveJob enabled
+          # Manual Offset management enabled
+          module Mom
+            # Standard ActiveJob strategy is the same one we use for Mom
+            include Strategies::Mom::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              manual_offset_management
+            ].freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Aj
+          # ActiveJob enabled
+          # Manual offset management enabled
+          # Virtual Partitions enabled
+          module MomVp
+            include Strategies::Default
+            include Strategies::Vp::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              active_job
+              manual_offset_management
+              virtual_partitions
+            ].freeze
+
+            # Standard flow without any features
+            def handle_after_consume
+              coordinator.on_finished do |last_group_message|
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  return if revoked?
+
+                  mark_as_consumed(last_group_message)
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro processing strategies namespace
+      module Strategies
+        # Base strategy for Pro
+        module Base
+          include ::Karafka::Processing::Strategies::Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/default.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # No features enabled.
+        # No manual offset management
+        # No long running jobs
+        # No virtual partitions
+        # Nothing. Just standard, automatic flow
+        module Default
+          include Base
+          include ::Karafka::Processing::Strategies::Default
+
+          # Apply strategy for a non-feature based flow
+          FEATURES = %i[].freeze
+
+          # No actions needed for the standard flow here
+          def handle_before_enqueue
+            nil
+          end
+
+          # Increment number of attempts per one "full" job. For all VP on a single topic partition
+          # this also should run once.
+          def handle_before_consume
+            coordinator.on_started do
+              coordinator.pause_tracker.increment
+            end
+          end
+
+          # Run the user consumption code
+          def handle_consume
+            # We should not run the work at all on a partition that was revoked
+            # This can happen primarily when an LRJ job gets to the internal worker queue and
+            # this partition is revoked prior processing.
+            unless revoked?
+              Karafka.monitor.instrument('consumer.consume', caller: self)
+              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+                consume
+              end
+            end
+
+            # Mark job as successful
+            coordinator.success!(self)
+          rescue StandardError => e
+            # If failed, mark as failed
+            coordinator.failure!(self, e)
+
+            # Re-raise so reported in the consumer
+            raise e
+          ensure
+            # We need to decrease number of jobs that this coordinator coordinates as it has
+            # finished
+            coordinator.decrement
+          end
+
+          # Standard flow without any features
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # Do not mark last message if pause happened. This prevents a scenario where pause
+                # is overridden upon rebalance by marking
+                return if coordinator.manual_pause?
+
+                mark_as_consumed(last_group_message)
+              else
+                retry_after_pause
+              end
+            end
+          end
+
+          # Standard
+          def handle_revoked
+            coordinator.on_revoked do
+              resume
+
+              coordinator.revoke
+            end
+
+            Karafka.monitor.instrument('consumer.revoke', caller: self)
+            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              revoked
+            end
+          end
+        end
+      end
+    end
+  end
+end