karafka 2.0.15 → 2.0.17

data/lib/karafka/pro/processing/strategies/dlq_lrj_mom.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Strategy for supporting DLQ with Mom and LRJ enabled
+        module DlqLrjMom
+          # This strategy needs to pause and revoke same way as DlqLrj but without the offset
+          # management
+          include DlqLrj
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            long_running_job
+            manual_offset_management
+          ].freeze
+
+          # LRJ standard flow after consumption with DLQ dispatch and no offset management
+          def handle_after_consume
+            coordinator.on_finished do
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              else
+                coordinator.pause_tracker.reset
+
+                skippable_message = find_skippable_message
+
+                unless revoked?
+                  dispatch_to_dlq(skippable_message)
+                  seek(coordinator.seek_offset)
+                end
+
+                resume
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq_mom.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Strategy for supporting DLQ with Mom enabled
+        module DlqMom
+          # The broken message lookup is the same in this scenario
+          include Dlq
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            manual_offset_management
+          ].freeze
+
+          # When manual offset management is on, we do not mark anything as consumed automatically
+          # and we rely on the user to figure things out
+          def handle_after_consume
+            coordinator.on_finished do
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                pause(coordinator.seek_offset)
+              # If we've reached number of retries that we could, we need to skip the first message
+              # that was not marked as consumed, pause and continue, while also moving this message
+              # to the dead topic.
+              #
+              # For a Mom setup, this means, that user has to manage the checkpointing by himself.
+              # If no checkpointing is ever done, we end up with an endless loop.
+              else
+                # We reset the pause to indicate we will now consider it as "ok".
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message)
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Long-Running Job enabled
+        module Lrj
+          include Default
+
+          # Pause for tops 31 years
+          MAX_PAUSE_TIME = 1_000_000_000_000
+
+          # Features for this strategy
+          FEATURES = %i[
+            long_running_job
+          ].freeze
+
+          # We always need to pause prior to doing any jobs for LRJ
+          def handle_before_enqueue
+            # This ensures that when running LRJ with VP, things operate as expected run only once
+            # for all the virtual partitions collectively
+            coordinator.on_enqueued do
+              # Pause at the first message in a batch. That way in case of a crash, we will not
+              # loose any messages.
+              #
+              # For VP it applies the same way and since VP cannot be used with MOM we should not
+              # have any edge cases here.
+              pause(coordinator.seek_offset, MAX_PAUSE_TIME)
+            end
+          end
+
+          # LRJ standard flow after consumption
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                mark_as_consumed(last_group_message) unless revoked?
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              else
+                # If processing failed, we need to pause
+                # For long running job this will overwrite the default never-ending pause and will
+                # cause the processing to keep going after the error backoff
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # We do not un-pause on revokations for LRJ
+          def handle_revoked
+            coordinator.on_revoked do
+              # We do not want to resume on revocation in case of a LRJ.
+              # For LRJ we resume after the successful processing or do a backoff pause in case of
+              # a failure. Double non-blocking resume could cause problems in coordination.
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj_mom.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        module LrjMom
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            long_running_job
+            manual_offset_management
+          ].freeze
+
+          # We always need to pause prior to doing any jobs for LRJ
+          def handle_before_enqueue
+            # This ensures that when running LRJ with VP, things operate as expected run only once
+            # for all the virtual partitions collectively
+            coordinator.on_enqueued do
+              # Pause at the first message in a batch. That way in case of a crash, we will not
+              # loose any messages.
+              #
+              # For VP it applies the same way and since VP cannot be used with MOM we should not
+              # have any edge cases here.
+              pause(coordinator.seek_offset, Lrj::MAX_PAUSE_TIME)
+            end
+          end
+
+          # No offset management, aside from that typical LRJ
+          def handle_after_consume
+            coordinator.on_finished do
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+
+          # We do not un-pause on revokations for LRJ
+          def handle_revoked
+            coordinator.on_revoked do
+              coordinator.revoke
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj_vp.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Long-Running Job enabled
+        # Virtual Partitions enabled
+        module LrjVp
+          # Same flow as the standard Lrj
+          include Lrj
+
+          # Features for this strategy
+          FEATURES = %i[
+            long_running_job
+            virtual_partitions
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/mom.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Manual offset management enabled
+        module Mom
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            manual_offset_management
+          ].freeze
+
+          # When mom is enabled, we do not mark messages as consumed after processing
+          def handle_after_consume
+            coordinator.on_finished do
+              return if revoked?
+
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+              else
+                pause(coordinator.seek_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/vp.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Just Virtual Partitions enabled
+        module Vp
+          # This flow is exactly the same as the default one because the default one is wrapper
+          # with `coordinator#on_finished`
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            virtual_partitions
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategy_selector.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Selector of appropriate processing strategy matching topic combinations
+      # When using Karafka Pro, there is a different set of strategies than for regular, as there
+      # are different features.
+      class StrategySelector
+        def initialize
+          # We load them once for performance reasons not to do too many lookups
+          @available_strategies = Strategies
+                                  .constants
+                                  .delete_if { |k| k == :Base }
+                                  .map { |k| Strategies.const_get(k) }
+        end
+
+        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+        #   the strategy
+        # @return [Module] module with proper strategy
+        def find(topic)
+          feature_set = features_map(topic)
+
+          @available_strategies.find do |strategy|
+            strategy::FEATURES.sort == feature_set
+          end || raise(Errors::StrategyNotFoundError, topic.name)
+        end
+
+        private
+
+        # Builds features map used to find matching processing strategy
+        #
+        # @param topic [Karafka::Routing::Topic]
+        # @return [Array<Symbol>]
+        def features_map(topic)
+          [
+            topic.active_job? ? :active_job : nil,
+            topic.long_running_job? ? :long_running_job : nil,
+            topic.manual_offset_management? ? :manual_offset_management : nil,
+            topic.virtual_partitions? ? :virtual_partitions : nil,
+            topic.dead_letter_queue? ? :dead_letter_queue : nil
+          ].compact.sort
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/{contracts → routing/features}/base.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -11,10 +13,11 @@
 
 module Karafka
   module Pro
-    # Namespace for Karafka Pro related contracts
-    module Contracts
-      # Base contract for Pro components contracts
-      class Base < ::Karafka::Contracts::Base
+    module Routing
+      module Features
+        # Base class for all the Pro features
+        class Base < Karafka::Routing::Features::Base
+        end
       end
     end
   end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class DeadLetterQueue < Base
+          # Extended rules for dead letter queue settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('pro_topic')
+            end
+
+            # Make sure that we don't use DLQ with VP
+            # Using those two would cause many issues because the offset within VP is not
+            # manageable so in scenarios where we would fail on the last message, we would move by
+            # one and try again and fail, and move by one and try again and fail and so on...
+            virtual do |data, errors|
+              next unless errors.empty?
+
+              dead_letter_queue = data[:dead_letter_queue]
+              virtual_partitions = data[:virtual_partitions]
+
+              next unless dead_letter_queue[:active]
+              next unless virtual_partitions[:active]
+
+              [[%i[dead_letter_queue], :not_with_virtual_partitions]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/{builder_extensions.rb → features/dead_letter_queue.rb}

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component.
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
 # All of the commercial components are present in the lib/karafka/pro directory of this
 # repository and their usage requires commercial license agreement.
 #
@@ -11,18 +13,13 @@
 
 module Karafka
   module Pro
-    # Pro routing components
     module Routing
-      # Routing extensions for builder to be able to validate Pro components correct usage
-      module BuilderExtensions
-        # Validate consumer groups with pro contracts
-        # @param block [Proc] routing defining block
-        def draw(&block)
-          super
-
-          each do |consumer_group|
-            ::Karafka::Pro::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
-          end
+      module Features
+        # Enhanced Dead Letter Queue
+        #
+        # This version behaves similar to the regular version but provides ordering warranties and
+        # allows for batch skipping
+        class DeadLetterQueue < Base
         end
       end
     end

data/lib/karafka/pro/routing/features/long_running_job/config.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs configuration
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/contract.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Rules around manual offset management settings
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('pro_topic')
+            end
+
+            nested(:long_running_job) do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class LongRunningJob < Base
+          # Long-Running Jobs topic API extensions
+          module Topic
+            # @param active [Boolean] do we want to enable long-running job feature for this topic
+            def long_running_job(active = false)
+              @long_running_job ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is a given job on a topic a long-running one
+            def long_running_job?
+              long_running_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus lrj
+            def to_h
+              super.merge(
+                long_running_job: long_running_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        # Long-Running Jobs feature config and DSL namespace.
+        #
+        # Long-Running Jobs allow you to run Karafka jobs beyond `max.poll.interval.ms`
+        class LongRunningJob < Base
+        end
+      end
+    end
+  end
+end