karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/consumer_group.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ParallelSegments < Base
+            # Parallel segments are defined on the consumer group (since it creates many), thus we
+            # define them on the consumer group.
+            # This module adds extra methods needed there to make it work
+            module ConsumerGroup
+              # @return [Config] parallel segments config
+              def parallel_segments
+                # We initialize it as disabled if not configured by the user
+                public_send(:parallel_segments=, count: 1)
+              end
+
+              # Allows setting parallel segments configuration
+              #
+              # @param count [Integer] number of parallel segments (number of parallel consumer
+              #   groups that will be created)
+              # @param partitioner [nil, #call] nil or callable partitioner
+              # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
+              #   reducer to achieve enhanced parallelization when the default reducer is not enough.
+              # @param merge_key [String] key used to build the parallel segment consumer groups
+              #
+              # @note This method is an assignor but the API is actually via the `#parallel_segments`
+              #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
+              #   for the end users.
+              def parallel_segments=(
+                count: 1,
+                partitioner: nil,
+                reducer: nil,
+                merge_key: "-parallel-"
+              )
+                @parallel_segments ||= Config.new(
+                  active: count > 1,
+                  count: count,
+                  partitioner: partitioner,
+                  reducer: reducer || ->(parallel_key) { parallel_key.to_s.sum % count },
+                  merge_key: merge_key
+                )
+              end
+
+              # @return [Boolean] are parallel segments active
+              def parallel_segments?
+                parallel_segments.active?
+              end
+
+              # @return [Integer] id of the segment (0 or bigger) or -1 if parallel segments are not
+              #   active
+              def segment_id
+                return @segment_id if @segment_id
+
+                @segment_id = if parallel_segments?
+                  name.split(parallel_segments.merge_key).last.to_i
+                else
+                  -1
+                end
+              end
+
+              # @return [String] original segment consumer group name
+              def segment_origin
+                name.split(parallel_segments.merge_key).first
+              end
+
+              # @return [Hash] consumer group setup with the parallel segments definition in it
+              def to_h
+                super.merge(
+                  parallel_segments: parallel_segments.to_h.merge(
+                    segment_id: segment_id
+                  )
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/contracts/consumer_group.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ParallelSegments < Base
+            # Namespace for parallel segments contracts
+            module Contracts
+              # Contract to validate configuration of the parallel segments feature
+              class ConsumerGroup < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
+
+                  nested(:parallel_segments) do
+                    required(:active) { |val| [true, false].include?(val) }
+                    required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
+                    required(:reducer) { |val| val.respond_to?(:call) }
+                    required(:count) { |val| val.is_a?(Integer) && val >= 1 }
+                    required(:merge_key) { |val| val.is_a?(String) && val.size >= 1 }
+                  end
+
+                  # When parallel segments are defined, partitioner needs to respond to `#call` and
+                  # it cannot be nil
+                  virtual do |data, errors|
+                    next unless errors.empty?
+
+                    parallel_segments = data[:parallel_segments]
+
+                    next unless parallel_segments[:active]
+                    next if parallel_segments[:partitioner].respond_to?(:call)
+
+                    [[%i[parallel_segments partitioner], :respond_to_call]]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{parallel_segments → consumer_groups/parallel_segments}/topic.rb

@@ -32,33 +32,35 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class ParallelSegments < Base
-          # Parallel segments related expansions to the topic building flow
-          module Topic
-            # Injects the parallel segments filter as the first filter during building of each of
-            # the topics in case parallel segments are enabled.
-            #
-            # @param args [Object] anything accepted by the topic initializer
-            def initialize(*args)
-              super
+        module ConsumerGroups
+          class ParallelSegments < Base
+            # Parallel segments related expansions to the topic building flow
+            module Topic
+              # Injects the parallel segments filter as the first filter during building of each of
+              # the topics in case parallel segments are enabled.
+              #
+              # @param args [Object] anything accepted by the topic initializer
+              def initialize(*args)
+                super
 
-              return unless consumer_group.parallel_segments?
+                return unless group.parallel_segments?
 
-              builder = lambda do |topic, _partition|
-                mom = topic.manual_offset_management?
+                builder = lambda do |topic, _partition|
+                  mom = topic.manual_offset_management?
 
-                # We have two filters for mom and non-mom scenario not to mix this logic
-                filter_scope = Karafka::Pro::Processing::ParallelSegments::Filters
-                filter_class = mom ? filter_scope::Mom : filter_scope::Default
+                  # We have two filters for mom and non-mom scenario not to mix this logic
+                  filter_scope = Karafka::Pro::Processing::ConsumerGroups::ParallelSegments::Filters
+                  filter_class = mom ? filter_scope::Mom : filter_scope::Default
 
-                filter_class.new(
-                  segment_id: consumer_group.segment_id,
-                  partitioner: consumer_group.parallel_segments.partitioner,
-                  reducer: consumer_group.parallel_segments.reducer
-                )
-              end
+                  filter_class.new(
+                    segment_id: group.segment_id,
+                    partitioner: group.parallel_segments.partitioner,
+                    reducer: group.parallel_segments.reducer
+                  )
+                end
 
-              filter(builder)
+                filter(builder)
+              end
             end
           end
         end

data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Feature that allows parallelizing message processing within a single consumer group by
+          # creating multiple consumer group instances. It enables processing messages from each
+          # partition in parallel by distributing them to separate consumer group instances based on
+          # a partitioning key. Useful for both CPU and IO bound operations.
+          #
+          # Each parallel segment operates as an independent consumer group instance, processing
+          # messages that are assigned to it based on the configured partitioner and reducer.
+          # This allows for better resource utilization and increased processing throughput without
+          # requiring changes to the topic's partition count.
+          class ParallelSegments < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{patterns → consumer_groups/patterns}/builder.rb

@@ -32,19 +32,21 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Patterns < Base
-          # Expansions for the routing builder
-          module Builder
-            # Allows us to define the simple routing pattern matching
-            #
-            # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
-            #   automatic-based named patterns
-            # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
-            #   the regexp if we used named patterns
-            # @param block [Proc]
-            def pattern(regexp_or_name, regexp = nil, &block)
-              consumer_group(default_group_id) do
-                pattern(regexp_or_name, regexp, &block)
+        module ConsumerGroups
+          class Patterns < Base
+            # Expansions for the routing builder
+            module Builder
+              # Allows us to define the simple routing pattern matching
+              #
+              # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+              #   automatic-based named patterns
+              # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+              #   the regexp if we used named patterns
+              # @param block [Proc]
+              def pattern(regexp_or_name, regexp = nil, &block)
+                consumer_group(default_group_id) do
+                  pattern(regexp_or_name, regexp, &block)
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/{offset_metadata/contracts/topic.rb → consumer_groups/patterns/config.rb}

@@ -32,21 +32,36 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class OffsetMetadata < Base
-          # Namespace for offset metadata feature contracts
-          module Contracts
-            # Contract to validate configuration of the expiring feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+        module ConsumerGroups
+          # Holds pattern info reference
+          # Type is set to:
+          #   `:regular` - in case patterns are not used and topic is just a regular existing topic
+          #                matched directly based on the name
+          #   `:discovered` - in case it is a real topic on which we started to listed
+          #   `:matcher` - represents a regular expression used by librdkafka
+          class Patterns < Base
+            # Config for pattern based topic Only pattern related topics are active in this context
+            Config = Struct.new(
+              :active,
+              :type,
+              :pattern,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+
+              # @return [Boolean] is this a matcher topic
+              def matcher?
+                type == :matcher
+              end
+
+              # @return [Boolean] is this a discovered topic
+              def discovered?
+                type == :discovered
               end
 
-              nested(:offset_metadata) do
-                required(:active) { |val| val == true }
-                required(:cache) { |val| [true, false].include?(val) }
-                required(:deserializer) { |val| val.respond_to?(:call) }
+              # @return [Boolean] is this a regular topic
+              def regular?
+                type == :regular
               end
             end
           end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/consumer_group.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Expansion of the consumer groups routing component to work with patterns
+            module ConsumerGroup
+              # @param args [Object] whatever consumer group accepts
+              def initialize(*args)
+                super
+                @patterns = Patterns.new([])
+              end
+
+              # @return [::Karafka::Pro::Routing::Features::ConsumerGroups::Patterns::Patterns] created patterns
+              def patterns
+                @patterns
+              end
+
+              # Creates the pattern for topic matching with appropriate virtual topic
+              # @param regexp_or_name [Symbol, String, Regexp] name of the pattern or regexp for
+              #   automatic-based named patterns
+              # @param regexp [Regexp, nil] nil if we use auto-generated name based on the regexp or
+              #   the regexp if we used named patterns
+              # @param block [Proc] appropriate underlying topic settings
+              def pattern=(regexp_or_name, regexp = nil, &block)
+                # This code allows us to have a nice nameless (automatic-named) patterns that do not
+                # have to be explicitly named. However if someone wants to use names for exclusions
+                # it can be done by providing both
+                if regexp_or_name.is_a?(Regexp)
+                  name = nil
+                  regexp = regexp_or_name
+                else
+                  name = regexp_or_name
+                end
+
+                pattern = Pattern.new(name, regexp, block)
+                virtual_topic = public_send(:topic=, pattern.name, &block)
+                # Indicate the nature of this topic (matcher)
+                virtual_topic.patterns(active: true, type: :matcher, pattern: pattern)
+                # Pattern subscriptions should never be part of declarative topics definitions
+                # Since they are subscribed by regular expressions, we do not know the target
+                # topics names so we cannot manage them via declaratives
+                virtual_topic.config(active: false)
+                pattern.topic = virtual_topic
+                @patterns << pattern
+              end
+
+              # @return [Hash] consumer group with patterns injected
+              def to_h
+                super.merge(
+                  patterns: patterns.map(&:to_h)
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/consumer_group.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            module Contracts
+              # Contract to validate configuration of the filtering feature
+              class ConsumerGroup < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
+
+                  required(:patterns) { |val| val.is_a?(Array) && val.all?(Hash) }
+
+                  virtual do |data, errors|
+                    next unless errors.empty?
+
+                    validator = Pattern.new
+
+                    data[:patterns].each do |pattern|
+                      validator.validate!(pattern)
+                    end
+
+                    nil
+                  end
+
+                  # Make sure, that there are no same regular expressions with different names
+                  # in a single consumer group
+                  virtual do |data, errors|
+                    next unless errors.empty?
+
+                    regexp_strings = data[:patterns].map { |pattern| pattern.fetch(:regexp_string) }
+                    regexp_names = data[:patterns].map { |pattern| pattern.fetch(:name) }
+
+                    # If all names are the same for the same regexp, it means its a multiplex and
+                    # we can allow it
+                    next if regexp_names.uniq.size == 1 && regexp_strings.uniq.size == 1
+                    next if regexp_strings.empty?
+                    next if regexp_strings.uniq.size == regexp_strings.size
+
+                    [[%i[patterns], :regexps_not_unique]]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{parallel_segments.rb → consumer_groups/patterns/contracts/pattern.rb}

@@ -32,16 +32,30 @@ module Karafka
   module Pro
     module Routing
       module Features
-        # Feature that allows parallelizing message processing within a single consumer group by
-        # creating multiple consumer group instances. It enables processing messages from each
-        # partition in parallel by distributing them to separate consumer group instances based on
-        # a partitioning key. Useful for both CPU and IO bound operations.
-        #
-        # Each parallel segment operates as an independent consumer group instance, processing
-        # messages that are assigned to it based on the configured partitioner and reducer.
-        # This allows for better resource utilization and increased processing throughput without
-        # requiring changes to the topic's partition count.
-        class ParallelSegments < Base
+        module ConsumerGroups
+          class Patterns < Base
+            # Namespace for patterns related contracts
+            module Contracts
+              # Contract used to validate pattern data
+              class Pattern < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("pattern")
+
+                  required(:regexp) { |val| val.is_a?(Regexp) }
+
+                  required(:regexp_string) do |val|
+                    val.is_a?(String) && val.start_with?("^") && val.size >= 2
+                  end
+
+                  required(:name) do |val|
+                    val.is_a?(String) && Karafka::Contracts::TOPIC_REGEXP.match?(val)
+                  end
+                end
+              end
+            end
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/consumer_groups/patterns/contracts/topic.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Patterns < Base
+            # Namespace for patterns feature contracts
+            module Contracts
+              # Contract to validate configuration of the patterns feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:patterns) do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:type) { |val| %i[matcher discovered regular].include?(val) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end