karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/multiplexing}/subscription_groups_builder.rb

@@ -32,21 +32,23 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Multiplexing < Base
-          # Expands the builder to multiply multiplexed groups
-          module SubscriptionGroupsBuilder
-            # Takes into consideration multiplexing and builds the more groups
-            #
-            # @param topics_array [Array<Routing::Topic>] group of topics that have the same
-            #   settings and can use the same connection
-            # @return [Array<Array<Routing::Topics>>] expanded groups
-            def expand(topics_array)
-              factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Expands the builder to multiply multiplexed groups
+            module SubscriptionGroupsBuilder
+              # Takes into consideration multiplexing and builds the more groups
+              #
+              # @param topics_array [Array<Routing::Topic>] group of topics that have the same
+              #   settings and can use the same connection
+              # @return [Array<Array<Routing::Topics>>] expanded groups
+              def expand(topics_array)
+                factor = topics_array.first.subscription_group_details.fetch(:multiplexing_max, 1)
 
-              Array.new(factor) do |i|
-                Karafka::Routing::Topics.new(
-                  i.zero? ? topics_array : topics_array.map(&:dup)
-                )
+                Array.new(factor) do |i|
+                  Karafka::Routing::Topics.new(
+                    i.zero? ? topics_array : topics_array.map(&:dup)
+                  )
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    # Namespace for Pro routing enhancements
+    module Routing
+      # Namespace for additional Pro features
+      module Features
+        module ConsumerGroups
+          # Multiplexing allows for creating multiple subscription groups for the same topic inside
+          # of the same subscription group allowing for better parallelism with limited number
+          # of processes
+          class Multiplexing < Base
+            class << self
+              # @param _config [Karafka::Core::Configurable::Node] app config node
+              def pre_setup(_config)
+                # Make sure we use proper unique validator for topics definitions
+                Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
+                  Patches::Contracts::ConsumerGroup
+                )
+              end
+
+              # If needed installs the needed listener and initializes tracker
+              #
+              # @param config [Karafka::Core::Configurable::Node]
+              def post_setup(config)
+                config.monitor.subscribe("app.before_warmup") do
+                  Contracts::Routing.new.validate!(
+                    config.internal.routing.builder,
+                    scope: %w[multiplexing]
+                  )
+                end
+
+                Karafka::App.monitor.subscribe("app.running") do
+                  # Do not install the manager and listener to control multiplexing unless there is
+                  # multiplexing enabled and it is dynamic.
+                  # We only need to control multiplexing when it is in a dynamic state
+                  next unless Karafka::App
+                    .subscription_groups
+                    .values
+                    .flat_map(&:itself)
+                    .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
+
+                  # Subscribe for events and possibility to manage via the Pro connection manager
+                  # that supports multiplexing
+                  Karafka.monitor.subscribe(
+                    Karafka::Pro::Connection::Multiplexing::Listener.new
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/non_blocking_job/topic.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class NonBlockingJob < Base
+            # Non-Blocking Jobs topic API extensions
+            module Topic
+              # Non blocking setting method
+              def non_blocking_job(*)
+                long_running_job(*)
+              end
+
+              alias_method :non_blocking, :non_blocking_job
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{non_blocking_job.rb → consumer_groups/non_blocking_job.rb}

@@ -34,19 +34,21 @@ module Karafka
     module Routing
       # Namespace for additional Pro features
       module Features
-        # Non Blocking Job is just an alias for LRJ.
-        #
-        # We however have it as a separate feature because its use-case may vary from LRJ.
-        #
-        # While LRJ is used mainly for long-running jobs that would take more than max poll
-        # interval time, non-blocking can be applied to make sure that we do not wait with polling
-        # of different partitions and topics that are subscribed together.
-        #
-        # This effectively allows for better resources utilization
-        #
-        # All the underlying code is the same but use-case is different and this should be
-        # reflected in the routing, hence this "virtual" feature.
-        class NonBlockingJob < Base
+        module ConsumerGroups
+          # Non Blocking Job is just an alias for LRJ.
+          #
+          # We however have it as a separate feature because its use-case may vary from LRJ.
+          #
+          # While LRJ is used mainly for long-running jobs that would take more than max poll
+          # interval time, non-blocking can be applied to make sure that we do not wait with polling
+          # of different partitions and topics that are subscribed together.
+          #
+          # This effectively allows for better resources utilization
+          #
+          # All the underlying code is the same but use-case is different and this should be
+          # reflected in the routing, hence this "virtual" feature.
+          class NonBlockingJob < Base
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/config.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class OffsetMetadata < Base
+            # Config for commit metadata feature
+            Config = Struct.new(
+              :active,
+              :deserializer,
+              :cache,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+              alias_method :cache?, :cache
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/contracts/topic.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class OffsetMetadata < Base
+            # Namespace for offset metadata feature contracts
+            module Contracts
+              # Contract to validate configuration of the expiring feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:offset_metadata) do
+                  required(:active) { |val| val == true }
+                  required(:cache) { |val| [true, false].include?(val) }
+                  required(:deserializer) { |val| val.respond_to?(:call) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/offset_metadata/topic.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # This feature allows for saving and retrieving offset metadata with custom deserialization
+          # support. It allows for storing extra data during commits that can be then used to alter
+          # the processing flow after a rebalance.
+          #
+          # @note Because this feature has zero performance impact and makes no queries to Kafka
+          #   unless requested, it is always enabled.
+          class OffsetMetadata < Base
+            # Empty string not to create it on each deserialization
+            EMPTY_STRING = ""
+
+            # Default deserializer just ensures we always get a string as without metadata by
+            # default it would be nil
+            STRING_DESERIALIZER = ->(raw_metadata) { raw_metadata || EMPTY_STRING }.freeze
+
+            private_constant :STRING_DESERIALIZER, :EMPTY_STRING
+
+            # Commit Metadata API extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @offset_metadata = nil
+                super
+              end
+
+              # @param cache [Boolean] should we cache the response until rebalance
+              # @param deserializer [#call] deserializer that will get raw data and should return
+              #   deserialized metadata
+              # @return [Config] this feature config
+              def offset_metadata(cache: true, deserializer: STRING_DESERIALIZER)
+                @offset_metadata ||= Config.new(
+                  active: true,
+                  cache: cache,
+                  deserializer: deserializer
+                )
+              end
+
+              # @return [true] is offset metadata active (it always is)
+              def offset_metadata?
+                offset_metadata.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus offset metadata
+              #   settings
+              def to_h
+                super.merge(
+                  offset_metadata: offset_metadata.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{offset_metadata.rb → consumer_groups/offset_metadata.rb}

@@ -32,21 +32,23 @@ module Karafka
   module Pro
     module Routing
       module Features
-        # Offset Metadata Support with a custom deserializer
-        class OffsetMetadata < Base
-          class << self
-            # If needed installs the needed listener and initializes tracker
-            #
-            # @param _config [Karafka::Core::Configurable::Node] app config
-            def post_setup(_config)
-              Karafka::App.monitor.subscribe("app.running") do
-                # Initialize the tracker prior to becoming multi-threaded
-                Karafka::Processing::InlineInsights::Tracker.instance
+        module ConsumerGroups
+          # Offset Metadata Support with a custom deserializer
+          class OffsetMetadata < Base
+            class << self
+              # If needed installs the needed listener and initializes tracker
+              #
+              # @param _config [Karafka::Core::Configurable::Node] app config
+              def post_setup(_config)
+                Karafka::App.monitor.subscribe("app.running") do
+                  # Initialize the tracker prior to becoming multi-threaded
+                  Karafka::Processing::ConsumerGroups::InlineInsights::Tracker.instance
 
-                # Subscribe to the statistics reports and collect them
-                Karafka.monitor.subscribe(
-                  Karafka::Pro::Processing::OffsetMetadata::Listener.new
-                )
+                  # Subscribe to the statistics reports and collect them
+                  Karafka.monitor.subscribe(
+                    Karafka::Pro::Processing::ConsumerGroups::OffsetMetadata::Listener.new
+                  )
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/consumer_groups/parallel_segments/builder.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ParallelSegments < Base
+            # Expansions for the routing builder
+            module Builder
+              # Builds and saves given consumer group
+              # @param group_id [String, Symbol] name for consumer group
+              # @param block [Proc] proc that should be executed in the proxy context
+              def consumer_group(group_id, &block)
+                existing = find { |group| group.name == group_id.to_s }
+
+                # Re-opening a CG should not change its parallel setup
+                if existing
+                  super
+                else
+                  # We build a temp group and a target to check if it has parallel segments
+                  # enabled and if so, we do not add it to the routing but instead we build the
+                  # appropriate number of parallel segment groups
+                  temp_group = Karafka::Routing::ConsumerGroup.new(group_id.to_s)
+                  temp_target = Karafka::Routing::Proxy.new(temp_group, &block).target
+                  config = temp_target.parallel_segments
+
+                  if config.active?
+                    config.count.times do |i|
+                      sub_name = [group_id, config.merge_key, i.to_s].join
+                      sub_group = Karafka::Routing::ConsumerGroup.new(sub_name)
+                      self << Karafka::Routing::Proxy.new(sub_group, &block).target
+                    end
+                  # If not parallel segments are not active we go with the default flow
+                  else
+                    super
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{multiplexing → consumer_groups/parallel_segments}/config.rb

@@ -32,21 +32,19 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Multiplexing < Base
-          # Multiplexing configuration
-          Config = Struct.new(
-            :active,
-            :min,
-            :max,
-            :boot,
-            :scale_delay,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-
-            # @return [Boolean] true if we are allowed to upscale and downscale
-            def dynamic?
-              min < max
+        module ConsumerGroups
+          class ParallelSegments < Base
+            # Config for parallel segments.
+            # @note Used on the consumer level, not per topic
+            Config = Struct.new(
+              :active,
+              :count,
+              :partitioner,
+              :reducer,
+              :merge_key,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
             end
           end
         end