karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/patterns/config.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Holds pattern info reference
-        # Type is set to:
-        #   `:regular` - in case patterns are not used and topic is just a regular existing topic
-        #                matched directly based on the name
-        #   `:discovered` - in case it is a real topic on which we started to listed
-        #   `:matcher` - represents a regular expression used by librdkafka
-        class Patterns < Base
-          # Config for pattern based topic
-          # Only pattern related topics are active in this context
-          Config = Struct.new(
-            :active,
-            :type,
-            :pattern,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-
-            # @return [Boolean] is this a matcher topic
-            def matcher?
-              type == :matcher
-            end
-
-            # @return [Boolean] is this a discovered topic
-            def discovered?
-              type == :discovered
-            end
-
-            # @return [Boolean] is this a regular topic
-            def regular?
-              type == :regular
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/consumer_group.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Expansion of the consumer groups routing component to work with patterns
-          module ConsumerGroup
-            # @param args [Object] whatever consumer group accepts
-            def initialize(*args)
-              super
-              @patterns = Patterns.new([])
-            end
-
-            # @return [::Karafka::Pro::Routing::Features::Patterns::Patterns] created patterns
-            def patterns
-              @patterns
-            end
-
-            # Creates the pattern for topic matching with appropriate virtual topic
-            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
-            #   automatic-based named patterns
-            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
-            #   the regexp if we used named patterns
-            # @param block [Proc] appropriate underlying topic settings
-            def pattern=(regexp_or_name, regexp = nil, &block)
-              # This code allows us to have a nice nameless (automatic-named) patterns that do not
-              # have to be explicitly named. However if someone wants to use names for exclusions
-              # it can be done by providing both
-              if regexp_or_name.is_a?(Regexp)
-                name = nil
-                regexp = regexp_or_name
-              else
-                name = regexp_or_name
-              end
-
-              pattern = Pattern.new(name, regexp, block)
-              virtual_topic = public_send(:topic=, pattern.name, &block)
-              # Indicate the nature of this topic (matcher)
-              virtual_topic.patterns(active: true, type: :matcher, pattern: pattern)
-              # Pattern subscriptions should never be part of declarative topics definitions
-              # Since they are subscribed by regular expressions, we do not know the target
-              # topics names so we cannot manage them via declaratives
-              virtual_topic.config(active: false)
-              pattern.topic = virtual_topic
-              @patterns << pattern
-            end
-
-            # @return [Hash] consumer group with patterns injected
-            def to_h
-              super.merge(
-                patterns: patterns.map(&:to_h)
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/contracts/topic.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Namespace for patterns feature contracts
-          module Contracts
-            # Contract to validate configuration of the patterns feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:patterns) do
-                required(:active) { |val| [true, false].include?(val) }
-                required(:type) { |val| %i[matcher discovered regular].include?(val) }
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/detector.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Detects if a given topic matches any of the patterns and if so, injects it into the
-          # given subscription group routing
-          #
-          # @note This is NOT thread-safe and should run in a thread-safe context that warranties
-          #   that there won't be any race conditions
-          class Detector
-            # Mutex for making sure that we do not modify same consumer group in runtime at the
-            # same time from multiple subscription groups if they operate in a multiplexed mode
-            MUTEX = Mutex.new
-
-            private_constant :MUTEX
-
-            # Checks if the provided topic matches any of the patterns and when detected, expands
-            # the routing with it.
-            #
-            # @param sg_topics [Array<Karafka::Routing::Topic>] given subscription group routing
-            #   topics.
-            # @param new_topic [String] new topic that we have detected
-            def expand(sg_topics, new_topic)
-              MUTEX.synchronize do
-                sg_topics
-                  .map(&:patterns)
-                  .select(&:active?)
-                  .select(&:matcher?)
-                  .map(&:pattern)
-                  .then { |pts| pts.empty? ? return : pts }
-                  .then { |pts| Patterns.new(pts) }
-                  .find(new_topic)
-                  .then { |pattern| pattern || return }
-                  .then { |pattern| install(pattern, new_topic, sg_topics) }
-              end
-            end
-
-            private
-
-            # Adds the discovered topic into the routing
-            #
-            # @param pattern [Karafka::Pro::Routing::Features::Patterns::Pattern] matched pattern
-            # @param discovered_topic [String] topic that we discovered that should be part of the
-            #   routing from now on.
-            # @param sg_topics [Array<Karafka::Routing::Topic>]
-            def install(pattern, discovered_topic, sg_topics)
-              consumer_group = pattern.topic.consumer_group
-
-              # Build new topic and register within the consumer group
-              topic = consumer_group.public_send(:topic=, discovered_topic, &pattern.config)
-              topic.patterns(active: true, type: :discovered)
-
-              # Assign the appropriate subscription group to this topic
-              topic.subscription_group = pattern.topic.subscription_group
-
-              # Inject into subscription group topics array always, so everything is reflected
-              # there but since it is not active, will not be picked
-              sg_topics << topic
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/pattern.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Karafka topic pattern object
-          # It represents a topic that is not yet materialized and that contains a name that is a
-          # regexp and not a "real" value. Underneath we define a dynamic topic, that is not
-          # active, that can be a subject to normal flow validations, etc.
-          class Pattern
-            # Pattern regexp
-            attr_accessor :regexp
-
-            # Each pattern has its own "topic" that we use as a routing reference that we define
-            # with non-existing topic for the routing to correctly pick it up for operations
-            # Virtual topic name for initial subscription
-            attr_reader :name
-
-            # Associated created virtual topic reference
-            attr_accessor :topic
-
-            # Config for real-topic configuration during injection
-            attr_reader :config
-
-            # @param name [String, Symbol, nil] name or the regexp for building the topic name or
-            #   nil if we want to make it based on the regexp content
-            # @param regexp [Regexp] regular expression to match topics
-            # @param config [Proc] config for topic bootstrap
-            def initialize(name, regexp, config)
-              @regexp = regexp
-              # This name is also used as the underlying matcher topic name
-              #
-              # It can be used provided by the user in case user wants to use exclusions of topics
-              # or we can generate it if irrelevant.
-              #
-              # We generate it based on the regexp so within the same consumer group they are
-              # always unique (checked by topic validations)
-              #
-              # This will not prevent users from creating a different regexps matching the same
-              # topic but this minimizes simple mistakes
-              #
-              # This sub-part of sh1 should be unique enough and short-enough to use it here
-              digest = Digest::SHA256.hexdigest(safe_regexp.source)[8..16]
-              @name = name ? name.to_s : "karafka-pattern-#{digest}"
-              @config = config
-            end
-
-            # @return [String] defined regexp representation as a string that is compatible with
-            #   librdkafka expectations. We use it as a subscription name for initial patterns
-            #   subscription start.
-            def regexp_string
-              "^#{safe_regexp.source}"
-            end
-
-            # @return [Hash] hash representation of this routing pattern
-            def to_h
-              {
-                regexp: regexp,
-                name: name,
-                regexp_string: regexp_string
-              }.freeze
-            end
-
-            private
-
-            # Since pattern building happens before validations and we rely internally on the fact
-            # that regexp is provided and nothing else, we here "sanitize" the regexp for our
-            # internal usage. Karafka will not run anyhow because our post-routing contracts will
-            # prevent it from running but internally in this component we need to ensure, that
-            # prior to the validations we operate on a regexp
-            #
-            # @return [Regexp] returns a regexp always even if what we've received was not a regexp
-            def safe_regexp
-              # This regexp will never match anything
-              regexp.is_a?(Regexp) ? regexp : /$a/
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/topic.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Patterns feature topic extensions
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @patterns = nil
-            end
-
-            # @return [String] subscription name or the regexp string representing matching of
-            #   new topics that should be detected.
-            def subscription_name
-              (patterns.active? && patterns.matcher?) ? patterns.pattern.regexp_string : super
-            end
-
-            # @param active [Boolean] is this topic active member of patterns
-            # @param type [Symbol] type of topic taking part in pattern matching
-            # @param pattern [Regexp] regular expression for matching
-            def patterns(active: false, type: :regular, pattern: nil)
-              @patterns ||= Config.new(active: active, type: type, pattern: pattern)
-            end
-
-            # @return [Boolean] is this topic a member of patterns
-            def patterns?
-              patterns.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus patterns
-            def to_h
-              super.merge(
-                patterns: patterns.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns/topics.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Patterns < Base
-          # Patterns feature topic extensions
-          module Topics
-            # Finds topic by its name in a more extensive way than the regular. Regular uses the
-            # pre-existing topics definitions. This extension also runs the expansion based on
-            # defined routing patterns (if any)
-            #
-            # If topic does not exist, it will try to run discovery in case there are patterns
-            # defined that would match it.
-            # This allows us to support lookups for newly appearing topics based on their regexp
-            # patterns.
-            #
-            # @param topic_name [String] topic name
-            # @return [Karafka::Routing::Topic]
-            # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
-            #   please create an issue.
-            #
-            # @note This method should not be used in context of finding multiple missing topics in
-            #   loops because it catches exceptions and attempts to expand routes. If this is used
-            #   in a loop for lookups on thousands of topics with detector expansion, this may
-            #   be slow. It should be used in the context where newly discovered topics are found
-            #   and should by design match a pattern. For quick lookups on batches of topics, it
-            #   is recommended to use a custom built lookup with conditional expander.
-            def find(topic_name)
-              super
-            rescue Karafka::Errors::TopicNotFoundError
-              Detector.new.expand(self, topic_name)
-
-              super
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/patterns.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Dynamic topics builder feature.
-        #
-        # Allows you to define patterns in routes that would then automatically subscribe and
-        # start consuming new topics.
-        #
-        # This feature works by injecting a topic that represents a regexp subscription (matcher)
-        # that at the same time holds the builder block for full config of a newly detected topic.
-        #
-        # We inject a virtual topic to hold settings but also to be able to run validations
-        # during boot to ensure consistency of the pattern base setup.
-        class Patterns < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/periodic_job/config.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class PeriodicJob < Base
-          # Config for periodics topics feature
-          Config = Struct.new(
-            :active,
-            :during_pause,
-            :during_retry,
-            :interval,
-            :materialized,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :during_pause?, :during_pause
-            alias_method :during_retry?, :during_retry
-            alias_method :materialized?, :materialized
-          end
-        end
-      end
-    end
-  end
-end