karafka 1.4.12 → 2.2.10

data/lib/karafka/pro/routing/features/patterns/consumer_group.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Expansion of the consumer groups routing component to work with patterns
+          module ConsumerGroup
+            # @param args [Object] whatever consumer group accepts
+            def initialize(*args)
+              super
+              @patterns = Patterns.new([])
+            end
+
+            # @return [::Karafka::Pro::Routing::Features::Patterns::Patterns] created patterns
+            def patterns
+              @patterns
+            end
+
+            # Creates the pattern for topic matching with appropriate virtual topic
+            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+            #   automatic-based named patterns
+            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+            #   the regexp if we used named patterns
+            # @param block [Proc] appropriate underlying topic settings
+            def pattern=(regexp_or_name, regexp = nil, &block)
+              # This code allows us to have a nice nameless (automatic-named) patterns that do not
+              # have to be explicitly named. However if someone wants to use names for exclusions
+              # it can be done by providing both
+              if regexp_or_name.is_a?(Regexp)
+                name = nil
+                regexp = regexp_or_name
+              else
+                name = regexp_or_name
+              end
+
+              pattern = Pattern.new(name, regexp, block)
+              virtual_topic = public_send(:topic=, pattern.name, &block)
+              # Indicate the nature of this topic (matcher)
+              virtual_topic.patterns(active: true, type: :matcher, pattern: pattern)
+              # Pattern subscriptions should never be part of declarative topics definitions
+              # Since they are subscribed by regular expressions, we do not know the target
+              # topics names so we cannot manage them via declaratives
+              virtual_topic.config(active: false)
+              pattern.topic = virtual_topic
+              @patterns << pattern
+            end
+
+            # @return [Hash] consumer group with patterns injected
+            def to_h
+              super.merge(
+                patterns: patterns.map(&:to_h)
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/consumer_group.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          module Contracts
+            # Contract to validate configuration of the filtering feature
+            class ConsumerGroup < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('consumer_group')
+
+                required(:patterns) { |val| val.is_a?(Array) && val.all? { |el| el.is_a?(Hash) } }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  validator = Pattern.new
+
+                  data[:patterns].each do |pattern|
+                    validator.validate!(pattern)
+                  end
+
+                  nil
+                end
+
+                # Make sure, that there are no same regular expressions with different names
+                # in a single consumer group
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+
+                  next if regexp_strings.empty?
+                  next if regexp_strings.uniq.size == regexp_strings.size
+
+                  [[%i[patterns], :regexps_not_unique]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/pattern.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns related contracts
+          module Contracts
+            # Contract used to validate pattern data
+            class Pattern < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('pattern')
+
+                required(:regexp) { |val| val.is_a?(Regexp) }
+
+                required(:regexp_string) do |val|
+                  val.is_a?(String) && val.start_with?('^') && val.size >= 2
+                end
+
+                required(:name) do |val|
+                  val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Namespace for patterns feature contracts
+          module Contracts
+            # Contract to validate configuration of the patterns feature
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+              end
+
+              nested(:patterns) do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:type) { |val| %i[matcher discovered regular].include?(val) }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Detects if a given topic matches any of the patterns and if so, injects it into the
+          # given subscription group routing
+          #
+          # @note This is NOT thread-safe and should run in a thread-safe context that warranties
+          #   that there won't be any race conditions
+          class Detector
+            # Checks if the provided topic matches any of the patterns and when detected, expands
+            # the routing with it.
+            #
+            # @param sg_topics [Array<Karafka::Routing::Topic>] given subscription group routing
+            #   topics.
+            # @param new_topic [String] new topic that we have detected
+            def expand(sg_topics, new_topic)
+              sg_topics
+                .map(&:patterns)
+                .select(&:active?)
+                .select(&:matcher?)
+                .map(&:pattern)
+                .then { |pts| pts.empty? ? return : pts }
+                .then { |pts| Patterns.new(pts) }
+                .find(new_topic)
+                .then { |pattern| pattern || return }
+                .then { |pattern| install(pattern, new_topic, sg_topics) }
+            end
+
+            private
+
+            # Adds the discovered topic into the routing
+            #
+            # @param pattern [Karafka::Pro::Routing::Features::Patterns::Pattern] matched pattern
+            # @param discovered_topic [String] topic that we discovered that should be part of the
+            #   routing from now on.
+            # @param sg_topics [Array<Karafka::Routing::Topic>]
+            def install(pattern, discovered_topic, sg_topics)
+              consumer_group = pattern.topic.consumer_group
+
+              # Build new topic and register within the consumer group
+              topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
+              topic.patterns(active: true, type: :discovered)
+
+              # Assign the appropriate subscription group to this topic
+              topic.subscription_group = pattern.topic.subscription_group
+
+              # Inject into subscription group topics array always, so everything is reflected
+              # there but since it is not active, will not be picked
+              sg_topics << topic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Karafka topic pattern object
+          # It represents a topic that is not yet materialized and that contains a name that is a
+          # regexp and not a "real" value. Underneath we define a dynamic topic, that is not
+          # active, that can be a subject to normal flow validations, etc.
+          class Pattern
+            # Pattern regexp
+            attr_accessor :regexp
+
+            # Each pattern has its own "topic" that we use as a routing reference that we define
+            # with non-existing topic for the routing to correctly pick it up for operations
+            # Virtual topic name for initial subscription
+            attr_reader :name
+
+            # Associated created virtual topic reference
+            attr_accessor :topic
+
+            # Config for real-topic configuration during injection
+            attr_reader :config
+
+            # @param name [String, Symbol, nil] name or the regexp for building the topic name or
+            #   nil if we want to make it based on the regexp content
+            # @param regexp [Regexp] regular expression to match topics
+            # @param config [Proc] config for topic bootstrap
+            def initialize(name, regexp, config)
+              @regexp = regexp
+              # This name is also used as the underlying matcher topic name
+              #
+              # It can be used provided by the user in case user wants to use exclusions of topics
+              # or we can generate it if irrelevant.
+              #
+              # We generate it based on the regexp so within the same consumer group they are
+              # always unique (checked by topic validations)
+              #
+              # This will not prevent users from creating a different regexps matching the same
+              # topic but this minimizes simple mistakes
+              #
+              # This sub-part of sh1 should be unique enough and short-enough to use it here
+              digest = Digest::SHA1.hexdigest(safe_regexp.source)[8..16]
+              @name = name ? name.to_s : "karafka-pattern-#{digest}"
+              @config = config
+            end
+
+            # @return [String] defined regexp representation as a string that is compatible with
+            #   librdkafka expectations. We use it as a subscription name for initial patterns
+            #   subscription start.
+            def regexp_string
+              "^#{safe_regexp.source}"
+            end
+
+            # @return [Hash] hash representation of this routing pattern
+            def to_h
+              {
+                regexp: regexp,
+                name: name,
+                regexp_string: regexp_string
+              }.freeze
+            end
+
+            private
+
+            # Since pattern building happens before validations and we rely internally on the fact
+            # that regexp is provided and nothing else, we here "sanitize" the regexp for our
+            # internal usage. Karafka will not run anyhow because our post-routing contracts will
+            # prevent it from running but internally in this component we need to ensure, that
+            # prior to the validations we operate on a regexp
+            #
+            # @return [Regexp] returns a regexp always even if what we've received was not a regexp
+            def safe_regexp
+              # This regexp will never match anything
+              regexp.is_a?(Regexp) ? regexp : /$a/
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/patterns.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Representation of groups of topics
+          class Patterns < ::Karafka::Routing::Topics
+            # Finds first pattern matching given topic name
+            #
+            # @param topic_name [String] topic name that may match a pattern
+            # @return [Karafka::Routing::Pattern, nil] pattern or nil if not found
+            # @note Please keep in mind, that there may be many patterns matching given topic name
+            #   and we always pick the first one (defined first)
+            def find(topic_name)
+              @accumulator.find { |pattern| pattern.regexp =~ topic_name }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/topic.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topic
+            # @return [String] subscription name or the regexp string representing matching of
+            #   new topics that should be detected.
+            def subscription_name
+              patterns.active? && patterns.matcher? ? patterns.pattern.regexp_string : super
+            end
+
+            # @param active [Boolean] is this topic active member of patterns
+            # @param type [Symbol] type of topic taking part in pattern matching
+            # @param pattern [Regexp] regular expression for matching
+            def patterns(active: false, type: :regular, pattern: nil)
+              @patterns ||= Config.new(active: active, type: type, pattern: pattern)
+            end
+
+            # @return [Boolean] is this topic a member of patterns
+            def patterns?
+              patterns.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus patterns
+            def to_h
+              super.merge(
+                patterns: patterns.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns/topics.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Patterns < Base
+          # Patterns feature topic extensions
+          module Topics
+            # Finds topic by its name in a more extensive way than the regular. Regular uses the
+            # pre-existing topics definitions. This extension also runs the expansion based on
+            # defined routing patterns (if any)
+            #
+            # If topic does not exist, it will try to run discovery in case there are patterns
+            # defined that would match it.
+            # This allows us to support lookups for newly appearing topics based on their regexp
+            # patterns.
+            #
+            # @param topic_name [String] topic name
+            # @return [Karafka::Routing::Topic]
+            # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+            #   please create an issue.
+            #
+            # @note This method should not be used in context of finding multiple missing topics in
+            #   loops because it catches exceptions and attempts to expand routes. If this is used
+            #   in a loop for lookups on thousands of topics with detector expansion, this may
+            #   be slow. It should be used in the context where newly discovered topics are found
+            #   and should by design match a pattern. For quick lookups on batches of topics, it
+            #   is recommended to use a custom built lookup with conditional expander.
+            def find(topic_name)
+              super
+            rescue Karafka::Errors::TopicNotFoundError
+              Detector.new.expand(self, topic_name)
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/patterns.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Dynamic topics builder feature.
+        #
+        # Allows you to define patterns in routes that would then automatically subscribe and
+        # start consuming new topics.
+        #
+        # This feature works by injecting a topic that represents a regexp subscription (matcher)
+        # that at the same time holds the builder block for full config of a newly detected topic.
+        #
+        # We inject a virtual topic to hold settings but also to be able to run validations
+        # during boot to ensure consistency of the pattern base setup.
+        class Patterns < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/contracts/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Namespace for pausing feature
+          module Contracts
+            # Contract to make sure, that the pause settings on a per topic basis are as expected
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load(
+                  File.read(
+                    File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                  )
+                ).fetch('en').fetch('validations').fetch('topic')
+
+                required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+                required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  pause_timeout = data.fetch(:pause_timeout)
+                  pause_max_timeout = data.fetch(:pause_max_timeout)
+
+                  next if pause_timeout <= pause_max_timeout
+
+                  [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Expansion allowing for a per topic pause strategy definitions
+          module Topic
+            # Allows for per-topic pausing strategy setting
+            #
+            # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
+            # @param max_timeout [Integer] what is the max timeout in case of an exponential
+            #   backoff (milliseconds)
+            # @param with_exponential_backoff [Boolean] should we use exponential backoff
+            #
+            # @note We do not construct here the nested config like we do with other routing
+            #   features, because this feature operates on the OSS layer by injection of values
+            #   and a nested config is not needed.
+            def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
+              self.pause_timeout = timeout if timeout
+              self.pause_max_timeout = max_timeout if max_timeout
+
+              return unless with_exponential_backoff
+
+              self.pause_with_exponential_backoff = with_exponential_backoff
+            end
+          end
+        end
+      end
+    end
+  end
+end