karafka 2.1.13 → 2.2.0

data/lib/karafka/pro/routing/features/filtering/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Filtering < Base
+          # Namespace for filtering feature contracts
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:filtering) do
+                required(:active) { |val| [true, false].include?(val) }
+
+                required(:factories) do |val|
+                  val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/long_running_job/{contract.rb → contracts/topic.rb}

@@ -16,18 +16,21 @@ module Karafka
     module Routing
       module Features
         class LongRunningJob < Base
-          # Rules around long-running job settings
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
+          # Namespace for LRJ contracts
+          module Contracts
+            # Rules around long-running job settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
 
-            nested(:long_running_job) do
-              required(:active) { |val| [true, false].include?(val) }
+              nested(:long_running_job) do
+                required(:active) { |val| [true, false].include?(val) }
+              end
             end
           end
         end

data/lib/karafka/pro/routing/features/{filtering/contract.rb → patterns/builder.rb}

@@ -15,22 +15,19 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Filtering < Base
-          # Contract to validate configuration of the filtering feature
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:filtering) do
-              required(:active) { |val| [true, false].include?(val) }
-
-              required(:factories) do |val|
-                val.is_a?(Array) && val.all? { |factory| factory.respond_to?(:call) }
+        class Patterns < Base
+          # Expansions for the routing builder
+          module Builder
+            # Allows us to define the simple routing pattern matching
+            #
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc]
+            def pattern(regexp_or_name, regexp = nil, &block)
+              consumer_group('app') do
+                pattern(regexp_or_name, regexp, &block)
               end
             end
           end

data/lib/karafka/pro/routing/features/patterns/config.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Holds pattern info reference
+        # Type is set to:
+        #   `:regular` - in case patterns are not used and topic is just a regular existing topic
+        #                matched directly based on the name
+        #   `:discovered` - in case it is a real topic on which we started to listed
+        #   `:matcher` - represents a regular expression used by librdkafka
+        class Patterns < Base
+          # Config for pattern based topic
+          # Only pattern related topics are active in this context
+          Config = Struct.new(
+            :active,
+            :type,
+            :pattern,
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+
+            # @return [Boolean] is this a matcher topic
+            def matcher?
+              type == :matcher
+            end
+
+            # @return [Boolean] is this a discovered topic
+            def discovered?
+              type == :discovered
+            end
+
+            # @return [Boolean] is this a regular topic
+            def regular?
+              type == :regular
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/consumer_group.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Expansion of the consumer groups routing component to work with patterns
+          module ConsumerGroup
+            # @param args [Object] whatever consumer group accepts
+            def initialize(*args)
+              super
+              @patterns = Patterns.new([])
+            end
+
+            # @return [::Karafka::Pro::Routing::Features::Patterns::Patterns] created patterns
+            def patterns
+              @patterns
+            end
+
+            # Creates the pattern for topic matching with appropriate virtual topic
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc] appropriate underlying topic settings
+            def pattern=(regexp_or_name, regexp = nil, &block)
+              # This code allows us to have a nice nameless (automatic-named) patterns that do not
+              # have to be explicitly named. However if someone wants to use names for exclusions
+              # it can be done by providing both
+              if regexp_or_name.is_a?(Regexp)
+                name = nil
+                regexp = regexp_or_name
+              else
+                name = regexp_or_name
+              end
+
+              pattern = Pattern.new(name, regexp, block)
+              virtual_topic = public_send(:topic=, pattern.name, &block)
+              # Indicate the nature of this topic (matcher)
+              virtual_topic.patterns(active: true, type: :matcher, pattern: pattern)
+              pattern.topic = virtual_topic
+              @patterns << pattern
+            end
+
+            # @return [Hash] consumer group with patterns injected
+            def to_h
+              super.merge(
+                patterns: patterns.map(&:to_h)
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  validator = Pattern.new
+
+                  data[:patterns].each do |pattern|
+                    validator.validate!(pattern)
+                  end
+
+                  nil
+                end
+
+                # Make sure, that there are no same regular expressions with different names
+                # in a single consumer group
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+
+                  next if regexp_strings.empty?
+                  next if regexp_strings.uniq.size == regexp_strings.size
+
+                  [[%i[patterns], :regexps_not_unique]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns related contracts
+          module Contracts
+            # Contract used to validate pattern data
+            class Pattern < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('pattern')
+
+                required(:regexp) { |val| val.is_a?(Regexp) }
+
+                required(:regexp_string) do |val|
+                  val.is_a?(String) && val.start_with?('^') && val.size >= 2
+                end
+
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns feature contracts
+          module Contracts
+            # Contract to validate configuration of the patterns feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:patterns) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:type) { |val| %i[matcher discovered regular].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Detects if a given topic matches any of the patterns and if so, injects it into the
+          # given subscription group routing
+          #
+          # @note This is NOT thread-safe and should run in a thread-safe context that warranties
+          #   that there won't be any race conditions
+          class Detector
+            # Checks if the provided topic matches any of the patterns and when detected, expands
+            # the routing with it.
+            #
+            # @param sg_topics [Array<Karafka::Routing::Topic>] given subscription group routing
+            #   topics.
+            # @param new_topic [String] new topic that we have detected
+            def expand(sg_topics, new_topic)
+              sg_topics
+                .map(&:patterns)
+                .select(&:active?)
+                .select(&:matcher?)
+                .map(&:pattern)
+                .then { |pts| pts.empty? ? return : pts }
+                .then { |pts| Patterns.new(pts) }
+                .find(new_topic)
+                .then { |pattern| pattern || return }
+                .then { |pattern| install(pattern, new_topic, sg_topics) }
+            end
+
+            private
+
+            # Adds the discovered topic into the routing
+            #
+            # @param pattern [Karafka::Pro::Routing::Features::Patterns::Pattern] matched pattern
+            # @param discovered_topic [String] topic that we discovered that should be part of the
+            #   routing from now on.
+            # @param sg_topics [Array<Karafka::Routing::Topic>]
+            def install(pattern, discovered_topic, sg_topics)
+              consumer_group = pattern.topic.consumer_group
+
+              # Build new topic and register within the consumer group
+              topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
+              topic.patterns(active: true, type: :discovered)
+
+              # Inject into subscription group topics array always, so everything is reflected
+              # there but since it is not active, will not be picked
+              sg_topics << topic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Karafka topic pattern object
+          # It represents a topic that is not yet materialized and that contains a name that is a
+          # regexp and not a "real" value. Underneath we define a dynamic topic, that is not
+          # active, that can be a subject to normal flow validations, etc.
+          class Pattern
+            # Pattern regexp
+            attr_accessor :regexp
+
+            # Each pattern has its own "topic" that we use as a routing reference that we define
+            # with non-existing topic for the routing to correctly pick it up for operations
+            # Virtual topic name for initial subscription
+            attr_reader :name
+
+            # Associated created virtual topic reference
+            attr_accessor :topic
+
+            # Config for real-topic configuration during injection
+            attr_reader :config
+
+            # @param name [String, Symbol, nil] name or the regexp for building the topic name or
+            #   nil if we want to make it based on the regexp content
+            # @param regexp [Regexp] regular expression to match topics
+            # @param config [Proc] config for topic bootstrap
+            def initialize(name, regexp, config)
+              @regexp = regexp
+              # This name is also used as the underlying matcher topic name
+              #
+              # It can be used provided by the user in case user wants to use exclusions of topics
+              # or we can generate it if irrelevant.
+              #
+              # We generate it based on the regexp so within the same consumer group they are
+              # always unique (checked by topic validations)
+              #
+              # This will not prevent users from creating a different regexps matching the same
+              # topic but this minimizes simple mistakes
+              #
+              # This sub-part of sh1 should be unique enough and short-enough to use it here
+              digest = Digest::SHA1.hexdigest(regexp.source)[8..16]
+              @name = name ? name.to_s : "karafka-pattern-#{digest}"
+              @config = config
+            end
+
+            # @return [String] defined regexp representation as a string that is compatible with
+            #   librdkafka expectations. We use it as a subscription name for initial patterns
+            #   subscription start.
+            def regexp_string
+              "^#{regexp.source}"
+            end
+
+            # @return [Hash] hash representation of this routing pattern
+            def to_h
+              {
+                regexp: regexp,
+                name: name,
+                regexp_string: regexp_string
+              }.freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{delaying/contract.rb → patterns/patterns.rb}

@@ -15,20 +15,17 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Delaying < Base
-          # Contract to validate configuration of the expiring feature
-          class Contract < Contracts::Base
-            configure do |config|
-              config.error_messages = YAML.safe_load(
-                File.read(
-                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
-                )
-              ).fetch('en').fetch('validations').fetch('topic')
-            end
-
-            nested(:delaying) do
-              required(:active) { |val| [true, false].include?(val) }
-              required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+        class Patterns < Base
+          # Representation of groups of topics
+          class Patterns < ::Karafka::Routing::Topics
+            # Finds first pattern matching given topic name
+            #
+            # @param topic_name [String] topic name that may match a pattern
+            # @return [Karafka::Routing::Pattern, nil] pattern or nil if not found
+            # @note Please keep in mind, that there may be many patterns matching given topic name
+            #   and we always pick the first one (defined first)
+            def find(topic_name)
+              @accumulator.find { |pattern| pattern.regexp =~ topic_name }
             end
           end
         end

data/lib/karafka/pro/routing/features/patterns/topic.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topic
+            # @return [String] subscription name or the regexp string representing matching of
+            #   new topics that should be detected.
+            def subscription_name
+              patterns.active? && patterns.matcher? ? patterns.pattern.regexp_string : super
+            end
+
+            # @param active [Boolean] is this topic active member of patterns
+            # @param type [Symbol] type of topic taking part in pattern matching
+            # @param pattern [Regexp] regular expression for matching
+            def patterns(active: false, type: :regular, pattern: nil)
+              @patterns ||= Config.new(active: active, type: type, pattern: pattern)
+            end
+
+            # @return [Boolean] is this topic a member of patterns
+            def patterns?
+              patterns.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus patterns
+            def to_h
+              super.merge(
+                patterns: patterns.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end