karafka 2.0.38 → 2.0.40

data/lib/karafka/pro/processing/strategies/mom/ftr.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        module Mom
+          # Filtering support for MoM
+          module Ftr
+            include Strategies::Ftr::Default
+            include Strategies::Mom::Default
+
+            # MoM + Ftr
+            FEATURES = %i[
+              filtering
+              manual_offset_management
+            ].freeze
+
+            # When mom is enabled, we do not mark messages as consumed after processing
+            # but we also need to keep in mind throttling here
+            def handle_after_consume
+              coordinator.on_finished do
+                return if revoked?
+
+                if coordinator.success?
+                  coordinator.pause_tracker.reset
+
+                  # Do not throttle if paused
+                  return if coordinator.manual_pause?
+
+                  handle_post_filtering
+                else
+                  retry_after_pause
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # VP starting strategies
+        module Vp
+          # Just Virtual Partitions enabled
+          module Default
+            # This flow is exactly the same as the default one because the default one is wrapper
+            # with `coordinator#on_finished`
+            include Strategies::Default
+
+            # Features for this strategy
+            FEATURES = %i[
+              virtual_partitions
+            ].freeze
+
+            # @return [Boolean] is the virtual processing collapsed in the context of given
+            #   consumer.
+            def collapsed?
+              coordinator.collapsed?
+            end
+
+            # @return [Boolean] true if any of virtual partition we're operating in the entangled
+            #   mode has already failed and we know we are failing collectively.
+            #   Useful for early stop to minimize number of things processed twice.
+            #
+            # @note We've named it `#failing?` instead of `#failure?`  because it aims to be used
+            #   from within virtual partitions where we want to have notion of collective failing
+            #   not just "local" to our processing. We "are" failing with other virtual partitions
+            #   raising an error, but locally we are still processing.
+            def failing?
+              coordinator.failure?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{strategies/lrj_vp.rb → strategies.rb}

@@ -14,20 +14,8 @@
 module Karafka
   module Pro
     module Processing
+      # Pro processing strategies namespace
       module Strategies
-        # Long-Running Job enabled
-        # Virtual Partitions enabled
-        module LrjVp
-          # Same flow as the standard Lrj
-          include Vp
-          include Lrj
-
-          # Features for this strategy
-          FEATURES = %i[
-            long_running_job
-            virtual_partitions
-          ].freeze
-        end
       end
     end
   end

data/lib/karafka/pro/processing/strategy_selector.rb

@@ -18,39 +18,65 @@ module Karafka
       # When using Karafka Pro, there is a different set of strategies than for regular, as there
       # are different features.
       class StrategySelector
+        attr_reader :strategies
+
+        # Strategies that we support in the Pro offering
+        # They can be combined
+        SUPPORTED_FEATURES = %i[
+          active_job
+          long_running_job
+          manual_offset_management
+          virtual_partitions
+          dead_letter_queue
+          filtering
+        ].freeze
+
         def initialize
+          # Preload the strategies
           # We load them once for performance reasons not to do too many lookups
-          @available_strategies = Strategies
-                                  .constants
-                                  .delete_if { |k| k == :Base }
-                                  .map { |k| Strategies.const_get(k) }
+          @strategies = find_all
         end
 
         # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
         #   the strategy
         # @return [Module] module with proper strategy
         def find(topic)
-          feature_set = features_map(topic)
+          feature_set = SUPPORTED_FEATURES.map do |feature_name|
+            topic.public_send("#{feature_name}?") ? feature_name : nil
+          end
 
-          @available_strategies.find do |strategy|
+          feature_set.compact!
+          feature_set.sort!
+
+          @strategies.find do |strategy|
             strategy::FEATURES.sort == feature_set
           end || raise(Errors::StrategyNotFoundError, topic.name)
         end
 
         private
 
-        # Builds features map used to find matching processing strategy
-        #
-        # @param topic [Karafka::Routing::Topic]
-        # @return [Array<Symbol>]
-        def features_map(topic)
-          [
-            topic.active_job? ? :active_job : nil,
-            topic.long_running_job? ? :long_running_job : nil,
-            topic.manual_offset_management? ? :manual_offset_management : nil,
-            topic.virtual_partitions? ? :virtual_partitions : nil,
-            topic.dead_letter_queue? ? :dead_letter_queue : nil
-          ].compact.sort
+        # @return [Array<Module>] all available strategies
+        def find_all
+          scopes = [Strategies]
+          modules = Strategies.constants
+
+          modules.each do |const|
+            scopes << Strategies.const_get(const)
+            modules += scopes.last.constants
+          end
+
+          scopes.flat_map do |scope|
+            modules.map do |const|
+              next if const == :FEATURES
+              next unless scope.const_defined?(const)
+
+              candidate = scope.const_get(const)
+
+              next unless candidate.const_defined?(:FEATURES)
+
+              candidate
+            end
+          end.uniq.compact
         end
       end
     end

data/lib/karafka/pro/{processing/strategies/aj_mom.rb → routing/features/delaying/config.rb}

@@ -13,19 +13,13 @@
 
 module Karafka
   module Pro
-    module Processing
-      module Strategies
-        # ActiveJob enabled
-        # Manual Offset management enabled
-        module AjMom
-          # Standard ActiveJob strategy is the same one we use for Mom
-          include Mom
-
-          # Features for this strategy
-          FEATURES = %i[
-            active_job
-            manual_offset_management
-          ].freeze
+    module Routing
+      module Features
+        class Delaying < Base
+          # Delaying feature configuration
+          Config = Struct.new(:active, :delay, keyword_init: true) do
+            alias_method :active?, :active
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/delaying/contract.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Contract to validate configuration of the expiring feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:delaying) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Delaying < Base
+          # Topic delaying API extensions
+          module Topic
+            # @param delay [Integer, nil] minimum age of a message we want to process
+            def delaying(delay = nil)
+              # Those settings are used for validation
+              @delaying ||= begin
+                config = Config.new(active: !delay.nil?, delay: delay)
+
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Delayer.new(delay) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#delaying` accepts
+            def delay_by(*args)
+              delaying(*args)
+            end
+
+            # @return [Boolean] is a given job delaying
+            def delaying?
+              delaying.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus delaying
+            def to_h
+              super.merge(
+                delaying: delaying.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/delaying.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Delaying allows us to delay processing of certain topics. This is useful when we want
+        # to for any reason to wait until processing data from a topic. It does not sleep and
+        # instead uses pausing to manage delays. This allows us to free up processing resources
+        # and not block the polling thread.
+        #
+        # Delaying is a virtual feature realized via the filters
+        class Delaying < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Expiring feature configuration
+          Config = Struct.new(:active, :ttl, keyword_init: true) do
+            alias_method :active?, :active
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/contract.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Contract to validate configuration of the expiring feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:expiring) do
+              required(:active) { |val| [true, false].include?(val) }
+              required(:ttl) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Expiring < Base
+          # Topic expiring API extensions
+          module Topic
+            # @param ttl [Integer, nil] maximum time in ms a message is considered alive
+            def expiring(ttl = nil)
+              # Those settings are used for validation
+              @expiring ||= begin
+                config = Config.new(active: !ttl.nil?, ttl: ttl)
+
+                if config.active?
+                  factory = ->(*) { Pro::Processing::Filters::Expirer.new(ttl) }
+                  filter(factory)
+                end
+
+                config
+              end
+            end
+
+            # Just an alias for nice API
+            #
+            # @param args [Array] Anything `#expiring` accepts
+            def expire_in(*args)
+              expiring(*args)
+            end
+
+            # @return [Boolean] is a given job expiring
+            def expiring?
+              expiring.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus expiring
+            def to_h
+              super.merge(
+                expiring: expiring.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/expiring.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Expiring allows us to filter out messages that are older than our expectation.
+        # This can also be done in a consumer, but applying the filtering prior to the jobs
+        # enqueuing allows us to improve operations with virtual partitions and limit the number of
+        # not necessary messages being ever seen
+        class Expiring < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature configuration
+          Config = Struct.new(:factories, keyword_init: true) do
+            # @return [Boolean] is this feature in use. Are any filters defined
+            def active?
+              !factories.empty?
+            end
+
+            # @return [Array<Object>] array of filters applicable to a topic partition
+            def filters
+              factories.map(&:call)
+            end
+
+            # @return [Hash] this config hash
+            def to_h
+              super.merge(active: active?)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/contract.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Contract to validate configuration of the filtering feature
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested(:filtering) do
+              required(:active) { |val| [true, false].include?(val) }
+
+              required(:factories) do |val|
+                val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/filtering/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Filtering feature topic extensions
+          module Topic
+            # @param factory [#call, nil] Callable that can produce new filters instances per
+            #   assigned topic partition. nil as default so this feature is disabled
+            def filter(factory = nil)
+              @filtering ||= Config.new(factories: [])
+              @filtering.factories << factory if factory
+              @filtering
+            end
+
+            # @param args [Array] Anything `#filter` accepts
+            # @return [Filtering::Config] alias to match the naming API for features
+            def filtering(*args)
+              filter(*args)
+            end
+
+            # @return [Boolean] is a given job throttled
+            def filtering?
+              filtering.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus throttling
+            def to_h
+              super.merge(
+                filtering: filtering.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end