karafka 2.1.12 → 2.2.0

data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns related contracts
+          module Contracts
+            # Contract used to validate pattern data
+            class Pattern < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('pattern')
+
+                required(:regexp) { |val| val.is_a?(Regexp) }
+
+                required(:regexp_string) do |val|
+                  val.is_a?(String) && val.start_with?('^') && val.size >= 2
+                end
+
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns feature contracts
+          module Contracts
+            # Contract to validate configuration of the patterns feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:patterns) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:type) { |val| %i[matcher discovered regular].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Detects if a given topic matches any of the patterns and if so, injects it into the
+          # given subscription group routing
+          #
+          # @note This is NOT thread-safe and should run in a thread-safe context that warranties
+          #   that there won't be any race conditions
+          class Detector
+            # Checks if the provided topic matches any of the patterns and when detected, expands
+            # the routing with it.
+            #
+            # @param sg_topics [Array<Karafka::Routing::Topic>] given subscription group routing
+            #   topics.
+            # @param new_topic [String] new topic that we have detected
+            def expand(sg_topics, new_topic)
+              sg_topics
+                .map(&:patterns)
+                .select(&:active?)
+                .select(&:matcher?)
+                .map(&:pattern)
+                .then { |pts| pts.empty? ? return : pts }
+                .then { |pts| Patterns.new(pts) }
+                .find(new_topic)
+                .then { |pattern| pattern || return }
+                .then { |pattern| install(pattern, new_topic, sg_topics) }
+            end
+
+            private
+
+            # Adds the discovered topic into the routing
+            #
+            # @param pattern [Karafka::Pro::Routing::Features::Patterns::Pattern] matched pattern
+            # @param discovered_topic [String] topic that we discovered that should be part of the
+            #   routing from now on.
+            # @param sg_topics [Array<Karafka::Routing::Topic>]
+            def install(pattern, discovered_topic, sg_topics)
+              consumer_group = pattern.topic.consumer_group
+
+              # Build new topic and register within the consumer group
+              topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
+              topic.patterns(active: true, type: :discovered)
+
+              # Inject into subscription group topics array always, so everything is reflected
+              # there but since it is not active, will not be picked
+              sg_topics << topic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Karafka topic pattern object
+          # It represents a topic that is not yet materialized and that contains a name that is a
+          # regexp and not a "real" value. Underneath we define a dynamic topic, that is not
+          # active, that can be a subject to normal flow validations, etc.
+          class Pattern
+            # Pattern regexp
+            attr_accessor :regexp
+
+            # Each pattern has its own "topic" that we use as a routing reference that we define
+            # with non-existing topic for the routing to correctly pick it up for operations
+            # Virtual topic name for initial subscription
+            attr_reader :name
+
+            # Associated created virtual topic reference
+            attr_accessor :topic
+
+            # Config for real-topic configuration during injection
+            attr_reader :config
+
+            # @param name [String, Symbol, nil] name or the regexp for building the topic name or
+            #   nil if we want to make it based on the regexp content
+            # @param regexp [Regexp] regular expression to match topics
+            # @param config [Proc] config for topic bootstrap
+            def initialize(name, regexp, config)
+              @regexp = regexp
+              # This name is also used as the underlying matcher topic name
+              #
+              # It can be used provided by the user in case user wants to use exclusions of topics
+              # or we can generate it if irrelevant.
+              #
+              # We generate it based on the regexp so within the same consumer group they are
+              # always unique (checked by topic validations)
+              #
+              # This will not prevent users from creating a different regexps matching the same
+              # topic but this minimizes simple mistakes
+              #
+              # This sub-part of sh1 should be unique enough and short-enough to use it here
+              digest = Digest::SHA1.hexdigest(regexp.source)[8..16]
+              @name = name ? name.to_s : "karafka-pattern-#{digest}"
+              @config = config
+            end
+
+            # @return [String] defined regexp representation as a string that is compatible with
+            #   librdkafka expectations. We use it as a subscription name for initial patterns
+            #   subscription start.
+            def regexp_string
+              "^#{regexp.source}"
+            end
+
+            # @return [Hash] hash representation of this routing pattern
+            def to_h
+              {
+                regexp: regexp,
+                name: name,
+                regexp_string: regexp_string
+              }.freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb}

@@ -15,20 +15,17 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Delaying < Base
-          # Contract to validate configuration of the expiring feature
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:delaying) do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+        class Patterns < Base
+          # Representation of groups of topics
+          class Patterns < ::Karafka::Routing::Topics
+            # Finds first pattern matching given topic name
+            #
+            # @param topic_name [String] topic name that may match a pattern
+            # @return [Karafka::Routing::Pattern, nil] pattern or nil if not found
+            # @note Please keep in mind, that there may be many patterns matching given topic name
+            #   and we always pick the first one (defined first)
+            def find(topic_name)
+              @accumulator.find { |pattern| pattern.regexp =~ topic_name }
             end
           end
         end

data/lib/karafka/pro/routing/features/patterns/topic.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topic
+            # @return [String] subscription name or the regexp string representing matching of
+            #   new topics that should be detected.
+            def subscription_name
+              patterns.active? && patterns.matcher? ? patterns.pattern.regexp_string : super
+            end
+
+            # @param active [Boolean] is this topic active member of patterns
+            # @param type [Symbol] type of topic taking part in pattern matching
+            # @param pattern [Regexp] regular expression for matching
+            def patterns(active: false, type: :regular, pattern: nil)
+              @patterns ||= Config.new(active: active, type: type, pattern: pattern)
+            end
+
+            # @return [Boolean] is this topic a member of patterns
+            def patterns?
+              patterns.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus patterns
+            def to_h
+              super.merge(
+                patterns: patterns.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/topics.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topics
+            # Finds topic by its name in a more extensive way than the regular. Regular uses the
+            # pre-existing topics definitions. This extension also runs the expansion based on
+            # defined routing patterns (if any)
+            #
+            # If topic does not exist, it will try to run discovery in case there are patterns
+            # defined that would match it.
+            # This allows us to support lookups for newly appearing topics based on their regexp
+            # patterns.
+            #
+            # @param topic_name [String] topic name
+            # @return [Karafka::Routing::Topic]
+            # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+            #   please create an issue.
+            #
+            # @note This method should not be used in context of finding multiple missing topics in
+            #   loops because it catches exceptions and attempts to expand routes. If this is used
+            #   in a loop for lookups on thousands of topics with detector expansion, this may
+            #   be slow. It should be used in the context where newly discovered topics are found
+            #   and should by design match a pattern. For quick lookups on batches of topics, it
+            #   is recommended to use a custom built lookup with conditional expander.
+            def find(topic_name)
+              super
+            rescue Karafka::Errors::TopicNotFoundError
+              Detector.new.expand(self, topic_name)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Dynamic topics builder feature.
+        #
+        # Allows you to define patterns in routes that would then automatically subscribe and
+        # start consuming new topics.
+        #
+        # This feature works by injecting a topic that represents a regexp subscription (matcher)
+        # that at the same time holds the builder block for full config of a newly detected topic.
+        #
+        # We inject a virtual topic to hold settings but also to be able to run validations
+        # during boot to ensure consistency of the pattern base setup.
+        class Patterns < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Namespace for pausing feature
+          module Contracts
+            # Contract to make sure, that the pause settings on a per topic basis are as expected
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+
+                required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  pause_timeout = data.fetch(:pause_timeout)
+                  pause_max_timeout = data.fetch(:pause_max_timeout)
+
+                  next if pause_timeout <= pause_max_timeout
+
+                  [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/throttling/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Throttling < Base
+          # Namespace for throttling contracts
+          module Contracts
+            # Rules around throttling settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:throttling) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:interval) { |val| val.is_a?(Integer) && val.positive? }
+                required(:limit) do |val|
+                  (val.is_a?(Integer) || val == Float::INFINITY) && val.positive?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/virtual_partitions/contracts/topic.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class VirtualPartitions < Base
+          # Namespace for VP contracts
+          module Contracts
+            # Rules around virtual partitions
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:virtual_partitions) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+              end
+
+              # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+              # cannot be nil
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                virtual_partitions = data[:virtual_partitions]
+
+                next unless virtual_partitions[:active]
+                next if virtual_partitions[:partitioner].respond_to?(:call)
+
+                [[%i[virtual_partitions partitioner], :respond_to_call]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/consumer_group.rb

@@ -55,7 +55,7 @@ module Karafka
       def subscription_group=(name = SubscriptionGroup.id, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        @current_subscription_group_id = name
+        @current_subscription_group_id = name.to_s
 
         Proxy.new(self, &block)
 

data/lib/karafka/routing/features/active_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # This feature validation contracts
+        module Contracts
+          # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+          # in order to be able to use active job routing
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              # One should not define active job jobs without ActiveJob being available for usage
+              next if Object.const_defined?('ActiveJob::Base')
+
+              [[%i[consumer], :active_job_missing]]
+            end
+
+            # ActiveJob needs to always run with manual offset management
+            # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+            # consumed jobs that did not run because of shutdown.
+            virtual do |data, errors|
+              next unless errors.empty?
+              next unless data[:active_job][:active]
+              next if data[:manual_offset_management][:active]
+
+              [[%i[manual_offset_management], :must_be_enabled]]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/active_job/proxy.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class ActiveJob < Base
+        # Routing proxy extensions for ActiveJob
+        module Proxy
+          include Builder
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/base/expander.rb

@@ -30,15 +30,21 @@ module Karafka
             scope = @scope
 
             Module.new do
-              # Runs validations related to this feature on a topic
+              # Runs validations related to this feature on a routing resources
               #
               # @param block [Proc] routing defining block
               define_method :draw do |&block|
                 result = super(&block)
 
                 each do |consumer_group|
+                  if scope::Contracts.const_defined?('ConsumerGroup', false)
+                    scope::Contracts::ConsumerGroup.new.validate!(consumer_group.to_h)
+                  end
+
+                  next unless scope::Contracts.const_defined?('Topic', false)
+
                   consumer_group.topics.each do |topic|
-                    scope::Contract.new.validate!(topic.to_h)
+                    scope::Contracts::Topic.new.validate!(topic.to_h)
                   end
                 end
 

data/lib/karafka/routing/features/base.rb

@@ -14,9 +14,11 @@ module Karafka
           # Extends topic and builder with given feature API
           def activate
             Topic.prepend(self::Topic) if const_defined?('Topic', false)
-            Proxy.prepend(self::Builder) if const_defined?('Builder', false)
+            Topics.prepend(self::Topics) if const_defined?('Topics', false)
+            ConsumerGroup.prepend(self::ConsumerGroup) if const_defined?('ConsumerGroup', false)
+            Proxy.prepend(self::Proxy) if const_defined?('Proxy', false)
             Builder.prepend(self::Builder) if const_defined?('Builder', false)
-            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contract', false)
+            Builder.prepend(Base::Expander.new(self)) if const_defined?('Contracts', false)
           end
 
           # Loads all the features and activates them