karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/patterns/detector.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Detects if a given topic matches any of the patterns and if so, injects it into the
+            # given subscription group routing
+            #
+            # @note This is NOT thread-safe and should run in a thread-safe context that warranties
+            #   that there won't be any race conditions
+            class Detector
+              # Mutex for making sure that we do not modify same consumer group in runtime at the
+              # same time from multiple subscription groups if they operate in a multiplexed mode
+              MUTEX = Mutex.new
+
+              private_constant :MUTEX
+
+              # Checks if the provided topic matches any of the patterns and when detected, expands
+              # the routing with it.
+              #
+              # @param sg_topics [Array<Karafka::Routing::Topic>] given subscription group routing
+              #   topics.
+              # @param new_topic [String] new topic that we have detected
+              def expand(sg_topics, new_topic)
+                MUTEX.synchronize do
+                  sg_topics
+                    .map(&:patterns)
+                    .select(&:active?)
+                    .select(&:matcher?)
+                    .map(&:pattern)
+                    .then { |pts| pts.empty? ? return : pts }
+                    .then { |pts| Patterns.new(pts) }
+                    .find(new_topic)
+                    .then { |pattern| pattern || return }
+                    .then { |pattern| install(pattern, new_topic, sg_topics) }
+                end
+              end
+
+              private
+
+              # Adds the discovered topic into the routing
+              #
+              # @param pattern [Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Pattern] matched pattern
+              # @param discovered_topic [String] topic that we discovered that should be part of the
+              #   routing from now on.
+              # @param sg_topics [Array<Karafka::Routing::Topic>]
+              def install(pattern, discovered_topic, sg_topics)
+                group = pattern.topic.group
+
+                # Build new topic and register within the owning group
+                topic = group.public_send(:topic=, discovered_topic, &pattern.config)
+                topic.patterns(active: true, type: :discovered)
+
+                # Assign the appropriate subscription group to this topic
+                topic.subscription_group = pattern.topic.subscription_group
+
+                # Inject into subscription group topics array always, so everything is reflected
+                # there but since it is not active, will not be picked
+                sg_topics << topic
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/pattern.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Karafka topic pattern object
+            # It represents a topic that is not yet materialized and that contains a name that is a
+            # regexp and not a "real" value. Underneath we define a dynamic topic, that is not
+            # active, that can be a subject to normal flow validations, etc.
+            class Pattern
+              # Pattern regexp
+              attr_accessor :regexp
+
+              # Each pattern has its own "topic" that we use as a routing reference that we define
+              # with non-existing topic for the routing to correctly pick it up for operations
+              # Virtual topic name for initial subscription
+              attr_reader :name
+
+              # Associated created virtual topic reference
+              attr_accessor :topic
+
+              # Config for real-topic configuration during injection
+              attr_reader :config
+
+              # @param name [String, Symbol, nil] name or the regexp for building the topic name or
+              #   nil if we want to make it based on the regexp content
+              # @param regexp [Regexp] regular expression to match topics
+              # @param config [Proc] config for topic bootstrap
+              def initialize(name, regexp, config)
+                @regexp = regexp
+                # This name is also used as the underlying matcher topic name
+                #
+                # It can be used provided by the user in case user wants to use exclusions of topics
+                # or we can generate it if irrelevant.
+                #
+                # We generate it based on the regexp so within the same consumer group they are
+                # always unique (checked by topic validations)
+                #
+                # This will not prevent users from creating a different regexps matching the same
+                # topic but this minimizes simple mistakes
+                #
+                # This sub-part of SHA-256 should be unique enough and short enough to use here
+                digest = Digest::SHA256.hexdigest(safe_regexp.source)[8..16]
+                @name = name ? name.to_s : "karafka-pattern-#{digest}"
+                @config = config
+              end
+
+              # @return [String] defined regexp representation as a string that is compatible with
+              #   librdkafka expectations. We use it as a subscription name for initial patterns
+              #   subscription start.
+              def regexp_string
+                "^#{safe_regexp.source}"
+              end
+
+              # @return [Hash] hash representation of this routing pattern
+              def to_h
+                {
+                  regexp: regexp,
+                  name: name,
+                  regexp_string: regexp_string
+                }.freeze
+              end
+
+              private
+
+              # Since pattern building happens before validations and we rely internally on the fact
+              # that regexp is provided and nothing else, we here "sanitize" the regexp for our
+              # internal usage. Karafka will not run anyhow because our post-routing contracts will
+              # prevent it from running but internally in this component we need to ensure, that
+              # prior to the validations we operate on a regexp
+              #
+              # @return [Regexp] returns a regexp always even if what we've received was not a regexp
+              def safe_regexp
+                # This regexp will never match anything
+                regexp.is_a?(Regexp) ? regexp : /$a/
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{recurring_tasks/contracts/topic.rb → consumer_groups/patterns/patterns.rb}

@@ -32,19 +32,18 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class RecurringTasks < Base
-          # Namespace for recurring tasks contracts
-          module Contracts
-            # Rules around recurring tasks settings
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:recurring_tasks) do
-                required(:active) { |val| [true, false].include?(val) }
+        module ConsumerGroups
+          class Patterns < Base
+            # Representation of groups of topics
+            class Patterns < Karafka::Routing::Topics
+              # Finds first pattern matching given topic name
+              #
+              # @param topic_name [String] topic name that may match a pattern
+              # @return [Karafka::Routing::Pattern, nil] pattern or nil if not found
+              # @note Please keep in mind, that there may be many patterns matching given topic name
+              #   and we always pick the first one (defined first)
+              def find(topic_name)
+                @accumulator.find { |pattern| pattern.regexp =~ topic_name }
               end
             end
           end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/topic.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Patterns feature topic extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @patterns = nil
+                super
+              end
+
+              # @return [String] subscription name or the regexp string representing matching of
+              #   new topics that should be detected.
+              def subscription_name
+                (patterns.active? && patterns.matcher?) ? patterns.pattern.regexp_string : super
+              end
+
+              # @param active [Boolean] is this topic active member of patterns
+              # @param type [Symbol] type of topic taking part in pattern matching
+              # @param pattern [Regexp] regular expression for matching
+              def patterns(active: false, type: :regular, pattern: nil)
+                @patterns ||= Config.new(active: active, type: type, pattern: pattern)
+              end
+
+              # @return [Boolean] is this topic a member of patterns
+              def patterns?
+                patterns.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus patterns
+              def to_h
+                super.merge(
+                  patterns: patterns.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/topics.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Patterns feature topic extensions
+            module Topics
+              # Finds topic by its name in a more extensive way than the regular. Regular uses the
+              # pre-existing topics definitions. This extension also runs the expansion based on
+              # defined routing patterns (if any)
+              #
+              # If topic does not exist, it will try to run discovery in case there are patterns
+              # defined that would match it.
+              # This allows us to support lookups for newly appearing topics based on their regexp
+              # patterns.
+              #
+              # @param topic_name [String] topic name
+              # @return [Karafka::Routing::Topic]
+              # @raise [Karafka::Errors::TopicNotFoundError] this should never happen. If you see it,
+              #   please create an issue.
+              #
+              # @note This method should not be used in context of finding multiple missing topics in
+              #   loops because it catches exceptions and attempts to expand routes. If this is used
+              #   in a loop for lookups on thousands of topics with detector expansion, this may
+              #   be slow. It should be used in the context where newly discovered topics are found
+              #   and should by design match a pattern. For quick lookups on batches of topics, it
+              #   is recommended to use a custom built lookup with conditional expander.
+              def find(topic_name)
+                super
+              rescue Karafka::Errors::TopicNotFoundError
+                Detector.new.expand(self, topic_name)
+
+                super
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/patterns.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Dynamic topics builder feature.
+          #
+          # Allows you to define patterns in routes that would then automatically subscribe and
+          # start consuming new topics.
+          #
+          # This feature works by injecting a topic that represents a regexp subscription (matcher)
+          # that at the same time holds the builder block for full config of a newly detected topic.
+          #
+          # We inject a virtual topic to hold settings but also to be able to run validations
+          # during boot to ensure consistency of the pattern base setup.
+          class Patterns < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/config.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class PeriodicJob < Base
+            # Config for periodics topics feature
+            Config = Struct.new(
+              :active,
+              :during_pause,
+              :during_retry,
+              :interval,
+              :materialized,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+              alias_method :during_pause?, :during_pause
+              alias_method :during_retry?, :during_retry
+              alias_method :materialized?, :materialized
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{direct_assignments/contracts/consumer_group.rb → consumer_groups/periodic_job/contracts/topic.rb}

@@ -32,29 +32,24 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class DirectAssignments < Base
-          module Contracts
-            # Contract to validate configuration of the direct assignments feature
-            class ConsumerGroup < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
-
-                virtual do |data, errors|
-                  next unless errors.empty?
-
-                  active = data[:topics].map do |topic|
-                    topic[:direct_assignments][:active]
-                  end
-
-                  # If none active we use standard subscriptions
-                  next if active.none?
-                  # If all topics from a given CG are using direct assignments that is expected
-                  # and allowed
-                  next if active.all?
+        module ConsumerGroups
+          class PeriodicJob < Base
+            # Namespace for periodics messages contracts
+            module Contracts
+              # Contract to validate configuration of the periodics feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
 
-                  [[%i[direct_assignments], :homogenous]]
+                nested(:periodic_job) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:interval) { |val| val.is_a?(Integer) && val >= 100 }
+                  required(:during_pause) { |val| [true, false].include?(val) }
+                  required(:during_retry) { |val| [true, false].include?(val) }
+                  required(:materialized) { |val| [true, false].include?(val) }
                 end
               end
             end

data/lib/karafka/pro/routing/features/consumer_groups/periodic_job/topic.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class PeriodicJob < Base
+            # Periodic topic action flows extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @periodic_job = nil
+                super
+              end
+
+              # Defines topic as periodic. Periodic topics consumers will invoke `#tick` with each
+              # poll where messages were not received.
+              # @param active [Boolean] should ticking happen for this topic assignments.
+              # @param interval [Integer] minimum interval to run periodic jobs on given topic.
+              # @param during_pause [Boolean, nil] Should periodic jobs run when partition is paused.
+              #   It is set to `nil` by default allowing for detection when this value is not
+              #   configured but should be built dynamically based on LRJ status.
+              # @param during_retry [Boolean, nil] Should we run when there was an error and we are
+              #   in a retry flow. Please note that for this to work, `during_pause` also needs to be
+              #   set to true as errors retry happens after pause.
+              def periodic_job(
+                active = false,
+                interval: nil,
+                during_pause: nil,
+                during_retry: nil
+              )
+                @periodic_job ||= begin
+                  # Set to active if any of the values was configured
+                  active = true unless interval.nil?
+                  active = true unless during_pause.nil?
+                  active = true unless during_retry.nil?
+                  # Default is not to retry during retry flow
+                  during_retry = false if during_retry.nil?
+
+                  # If no interval, use default
+                  interval ||= Karafka::App.config.internal.tick_interval
+
+                  Config.new(
+                    active: active,
+                    interval: interval,
+                    during_pause: during_pause,
+                    during_retry: during_retry,
+                    # This is internal setting for state management, not part of the configuration
+                    # Do not overwrite.
+                    # If `during_pause` is explicit, we do not select it based on LRJ setup and we
+                    # consider if fully ready out of the box
+                    materialized: !during_pause.nil?
+                  )
+                end
+
+                return @periodic_job if @periodic_job.materialized?
+                return @periodic_job unless @long_running_job
+
+                # If not configured in any way, we want not to process during pause for LRJ.
+                # LRJ pauses by default when processing and during this time we do not want to
+                # tick at all. This prevents us from running periodic jobs while LRJ jobs are
+                # running. This of course has a side effect of not running when paused for any
+                # other reason but it is a compromise in the default settings
+                @periodic_job.during_pause = !long_running_job?
+                @periodic_job.materialized = true
+
+                @periodic_job
+              end
+
+              alias_method :periodic, :periodic_job
+
+              # @return [Boolean] is periodics active
+              def periodic_job?
+                periodic_job.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus periodics flows
+              #   settings
+              def to_h
+                super.merge(
+                  periodic_job: periodic_job.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end