karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/multiplexing/contracts/topic.rb

@@ -1,146 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Multiplexing < Base
-          # Namespace for multiplexing feature contracts
-          module Contracts
-            # Validates the subscription group multiplexing setup
-            # We validate it on the topic level as subscription groups are not built during the
-            # routing as they are pre-run dynamically built.
-            #
-            # multiplexing attributes are optional since multiplexing may not be enabled
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:subscription_group_details) do
-                optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
-                optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
-                optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
-                optional(:multiplexing_scale_delay) { |val| val.is_a?(Integer) && val >= 1_000 }
-              end
-
-              # Makes sure min is not more than max
-              virtual do |data, errors|
-                next unless errors.empty?
-                next unless min(data)
-                next unless max(data)
-
-                min = min(data)
-                max = max(data)
-
-                next if min <= max
-
-                [[%w[subscription_group_details], :multiplexing_min_max_mismatch]]
-              end
-
-              # Makes sure, that boot is between min and max
-              virtual do |data, errors|
-                next unless errors.empty?
-                next unless min(data)
-                next unless max(data)
-                next unless boot(data)
-
-                min = min(data)
-                max = max(data)
-                boot = boot(data)
-
-                next if boot.between?(min, max)
-
-                [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
-              end
-
-              # Makes sure, that boot is equal to min and max when not in dynamic mode
-              virtual do |data, errors|
-                next unless errors.empty?
-                next unless min(data)
-                next unless max(data)
-                next unless boot(data)
-
-                min = min(data)
-                max = max(data)
-                boot = boot(data)
-
-                # In dynamic mode there are other rules to check boot
-                next if min != max
-                next if boot == min
-
-                [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
-              end
-
-              # Makes sure we do not run multiplexing with 1 always which does not make much sense
-              # because then it behaves like without multiplexing and can create problems for
-              # users running multiplexed subscription groups with multiple topics
-              virtual do |data, errors|
-                next unless errors.empty?
-                next unless min(data)
-                next unless max(data)
-
-                min = min(data)
-                max = max(data)
-
-                next unless min == 1 && max == 1
-
-                [[%w[subscription_group_details], :multiplexing_one_not_enough]]
-              end
-
-              class << self
-                # @param data [Hash] topic details
-                # @return [Integer, false] min or false if missing
-                def min(data)
-                  data[:subscription_group_details].fetch(:multiplexing_min, false)
-                end
-
-                # @param data [Hash] topic details
-                # @return [Integer, false] max or false if missing
-                def max(data)
-                  data[:subscription_group_details].fetch(:multiplexing_max, false)
-                end
-
-                # @param data [Hash] topic details
-                # @return [Integer, false] boot or false if missing
-                def boot(data)
-                  data[:subscription_group_details].fetch(:multiplexing_boot, false)
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/multiplexing/proxy.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Multiplexing < Base
-          # Allows for multiplexing setup inside a consumer group definition
-          module Proxy
-            # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
-            #   disabling dynamic multiplexing
-            # @param max [Integer] max multiplexing count
-            # @param boot [Integer] how many listeners should we start during boot by default
-            # @param scale_delay [Integer] number of ms of delay before applying any scale
-            #   operation to a consumer group
-            def multiplexing(min: nil, max: 1, boot: nil, scale_delay: 60_000)
-              @target.current_subscription_group_details.merge!(
-                multiplexing_min: min || max,
-                multiplexing_max: max,
-                # Picks half of max by default as long as possible. Otherwise goes with min
-                multiplexing_boot: boot || [min || max, (max / 2)].max,
-                multiplexing_scale_delay: scale_delay
-              )
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/multiplexing.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    # Namespace for Pro routing enhancements
-    module Routing
-      # Namespace for additional Pro features
-      module Features
-        # Multiplexing allows for creating multiple subscription groups for the same topic inside
-        # of the same subscription group allowing for better parallelism with limited number
-        # of processes
-        class Multiplexing < Base
-          class << self
-            # @param _config [Karafka::Core::Configurable::Node] app config node
-            def pre_setup(_config)
-              # Make sure we use proper unique validator for topics definitions
-              Karafka::Routing::Contracts::ConsumerGroup.singleton_class.prepend(
-                Patches::Contracts::ConsumerGroup
-              )
-            end
-
-            # If needed installs the needed listener and initializes tracker
-            #
-            # @param config [Karafka::Core::Configurable::Node]
-            def post_setup(config)
-              config.monitor.subscribe("app.before_warmup") do
-                Contracts::Routing.new.validate!(
-                  config.internal.routing.builder,
-                  scope: %w[multiplexing]
-                )
-              end
-
-              Karafka::App.monitor.subscribe("app.running") do
-                # Do not install the manager and listener to control multiplexing unless there is
-                # multiplexing enabled and it is dynamic.
-                # We only need to control multiplexing when it is in a dynamic state
-                next unless Karafka::App
-                  .subscription_groups
-                  .values
-                  .flat_map(&:itself)
-                  .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
-
-                # Subscribe for events and possibility to manage via the Pro connection manager
-                # that supports multiplexing
-                Karafka.monitor.subscribe(
-                  Karafka::Pro::Connection::Multiplexing::Listener.new
-                )
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/offset_metadata/config.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class OffsetMetadata < Base
-          # Config for commit metadata feature
-          Config = Struct.new(
-            :active,
-            :deserializer,
-            :cache,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-            alias_method :cache?, :cache
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/offset_metadata/topic.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # This feature allows for saving and retrieving offset metadata with custom deserialization
-        # support. It allows for storing extra data during commits that can be then used to alter
-        # the processing flow after a rebalance.
-        #
-        # @note Because this feature has zero performance impact and makes no queries to Kafka
-        #   unless requested, it is always enabled.
-        class OffsetMetadata < Base
-          # Empty string not to create it on each deserialization
-          EMPTY_STRING = ""
-
-          # Default deserializer just ensures we always get a string as without metadata by
-          # default it would be nil
-          STRING_DESERIALIZER = ->(raw_metadata) { raw_metadata || EMPTY_STRING }.freeze
-
-          private_constant :STRING_DESERIALIZER, :EMPTY_STRING
-
-          # Commit Metadata API extensions
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @offset_metadata = nil
-            end
-
-            # @param cache [Boolean] should we cache the response until rebalance
-            # @param deserializer [#call] deserializer that will get raw data and should return
-            #   deserialized metadata
-            # @return [Config] this feature config
-            def offset_metadata(cache: true, deserializer: STRING_DESERIALIZER)
-              @offset_metadata ||= Config.new(
-                active: true,
-                cache: cache,
-                deserializer: deserializer
-              )
-            end
-
-            # @return [true] is offset metadata active (it always is)
-            def offset_metadata?
-              offset_metadata.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus offset metadata
-            #   settings
-            def to_h
-              super.merge(
-                offset_metadata: offset_metadata.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/parallel_segments/builder.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class ParallelSegments < Base
-          # Expansions for the routing builder
-          module Builder
-            # Builds and saves given consumer group
-            # @param group_id [String, Symbol] name for consumer group
-            # @param block [Proc] proc that should be executed in the proxy context
-            def consumer_group(group_id, &block)
-              consumer_group = find { |cg| cg.name == group_id.to_s }
-
-              # Re-opening a CG should not change its parallel setup
-              if consumer_group
-                super
-              else
-                # We build a temp consumer group and a target to check if it has parallel segments
-                # enabled and if so, we do not add it to the routing but instead we build the
-                # appropriate number of parallel segment groups
-                temp_consumer_group = Karafka::Routing::ConsumerGroup.new(group_id.to_s)
-                temp_target = Karafka::Routing::Proxy.new(temp_consumer_group, &block).target
-                config = temp_target.parallel_segments
-
-                if config.active?
-                  config.count.times do |i|
-                    sub_name = [group_id, config.merge_key, i.to_s].join
-                    sub_consumer_group = Karafka::Routing::ConsumerGroup.new(sub_name)
-                    self << Karafka::Routing::Proxy.new(sub_consumer_group, &block).target
-                  end
-                # If not parallel segments are not active we go with the default flow
-                else
-                  super
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/parallel_segments/config.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class ParallelSegments < Base
-          # Config for parallel segments.
-          # @note Used on the consumer level, not per topic
-          Config = Struct.new(
-            :active,
-            :count,
-            :partitioner,
-            :reducer,
-            :merge_key,
-            keyword_init: true
-          ) do
-            alias_method :active?, :active
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/parallel_segments/consumer_group.rb

@@ -1,108 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class ParallelSegments < Base
-          # Parallel segments are defined on the consumer group (since it creates many), thus we
-          # define them on the consumer group.
-          # This module adds extra methods needed there to make it work
-          module ConsumerGroup
-            # @return [Config] parallel segments config
-            def parallel_segments
-              # We initialize it as disabled if not configured by the user
-              public_send(:parallel_segments=, count: 1)
-            end
-
-            # Allows setting parallel segments configuration
-            #
-            # @param count [Integer] number of parallel segments (number of parallel consumer
-            #   groups that will be created)
-            # @param partitioner [nil, #call] nil or callable partitioner
-            # @param reducer [nil, #call] reducer for parallel key. It allows for using a custom
-            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
-            # @param merge_key [String] key used to build the parallel segment consumer groups
-            #
-            # @note This method is an assignor but the API is actually via the `#parallel_segments`
-            #   method. Our `Routing::Proxy` normalizes that the way we want to have it exposed
-            #   for the end users.
-            def parallel_segments=(
-              count: 1,
-              partitioner: nil,
-              reducer: nil,
-              merge_key: "-parallel-"
-            )
-              @parallel_segments ||= Config.new(
-                active: count > 1,
-                count: count,
-                partitioner: partitioner,
-                reducer: reducer || ->(parallel_key) { parallel_key.to_s.sum % count },
-                merge_key: merge_key
-              )
-            end
-
-            # @return [Boolean] are parallel segments active
-            def parallel_segments?
-              parallel_segments.active?
-            end
-
-            # @return [Integer] id of the segment (0 or bigger) or -1 if parallel segments are not
-            #   active
-            def segment_id
-              return @segment_id if @segment_id
-
-              @segment_id = if parallel_segments?
-                name.split(parallel_segments.merge_key).last.to_i
-              else
-                -1
-              end
-            end
-
-            # @return [String] original segment consumer group name
-            def segment_origin
-              name.split(parallel_segments.merge_key).first
-            end
-
-            # @return [Hash] consumer group setup with the parallel segments definition in it
-            def to_h
-              super.merge(
-                parallel_segments: parallel_segments.to_h.merge(
-                  segment_id: segment_id
-                )
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end