karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/inline_insights.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Enhanced inline insights
+          # Allows you to set up an automatic filter that will ensure, that metrics are always
+          # available when processing starts.
+          class InlineInsights < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/config.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class LongRunningJob < Base
+            # Long-Running Jobs configuration
+            Config = Struct.new(
+              :active,
+              keyword_init: true
+            ) { alias_method :active?, :active }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{multiplexing/patches/contracts/consumer_group.rb → consumer_groups/long_running_job/contracts/topic.rb}

@@ -32,22 +32,20 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Multiplexing < Base
-          # Patches to Karafka OSS
-          module Patches
-            # Contracts patches
+        module ConsumerGroups
+          class LongRunningJob < Base
+            # Namespace for LRJ contracts
             module Contracts
-              # Consumer group contract patches
-              module ConsumerGroup
-                # Redefines the setup allowing for multiple sgs as long as with different names
-                #
-                # @param topic [Hash] topic config hash
-                # @return [Array] topic unique key for validators
-                def topic_unique_key(topic)
-                  [
-                    topic[:name],
-                    topic[:subscription_group_details]
-                  ]
+              # Rules around long-running job settings
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:long_running_job) do
+                  required(:active) { |val| [true, false].include?(val) }
                 end
               end
             end

data/lib/karafka/pro/routing/features/consumer_groups/long_running_job/topic.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class LongRunningJob < Base
+            # Long-Running Jobs topic API extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @long_running_job = nil
+                super
+              end
+
+              # @param active [Boolean] do we want to enable long-running job feature for this topic
+              def long_running_job(active = false)
+                @long_running_job ||= Config.new(active: active)
+              end
+
+              alias_method :long_running, :long_running_job
+
+              # @return [Boolean] is a given job on a topic a long-running one
+              def long_running_job?
+                long_running_job.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus lrj
+              def to_h
+                super.merge(
+                  long_running_job: long_running_job.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{long_running_job.rb → consumer_groups/long_running_job.rb}

@@ -34,10 +34,12 @@ module Karafka
     module Routing
       # Namespace for additional Pro features
       module Features
-        # Long-Running Jobs feature config and DSL namespace.
-        #
-        # Long-Running Jobs allow you to run Karafka jobs beyond `max.poll.interval.ms`
-        class LongRunningJob < Base
+        module ConsumerGroups
+          # Long-Running Jobs feature config and DSL namespace.
+          #
+          # Long-Running Jobs allow you to run Karafka jobs beyond `max.poll.interval.ms`
+          class LongRunningJob < Base
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/config.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Multiplexing configuration
+            Config = Struct.new(
+              :active,
+              :min,
+              :max,
+              :boot,
+              :scale_delay,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+
+              # @return [Boolean] true if we are allowed to upscale and downscale
+              def dynamic?
+                min < max
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/routing.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Multiplexing < Base
+            module Contracts
+              # Contract that validates that dynamic multiplexing is not used with statistics
+              # disabled. Dynamic multiplexing relies on statistics emitted events for scaling
+              # decisions and will not function properly without them.
+              class Routing < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing")
+                end
+
+                # @param _builder [Karafka::Routing::Builder]
+                # @param scope [Array<String>]
+                def validate!(_builder, scope: [])
+                  has_dynamic = Karafka::App
+                    .subscription_groups
+                    .values
+                    .flat_map(&:itself)
+                    .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
+
+                  return unless has_dynamic
+
+                  stats_interval = App.config.kafka[:"statistics.interval.ms"]
+
+                  return unless stats_interval
+                  return unless stats_interval.zero?
+
+                  super(
+                    { dynamic_multiplexing_with_statistics_disabled: true },
+                    scope: scope
+                  )
+                end
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next unless data[:dynamic_multiplexing_with_statistics_disabled]
+
+                  [[%i[multiplexing], :statistics_required_for_dynamic_multiplexing]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/contracts/topic.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Namespace for multiplexing feature contracts
+            module Contracts
+              # Validates the subscription group multiplexing setup
+              # We validate it on the topic level as subscription groups are not built during the
+              # routing as they are pre-run dynamically built.
+              #
+              # multiplexing attributes are optional since multiplexing may not be enabled
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:subscription_group_details) do
+                  optional(:multiplexing_min) { |val| val.is_a?(Integer) && val >= 1 }
+                  optional(:multiplexing_max) { |val| val.is_a?(Integer) && val >= 1 }
+                  optional(:multiplexing_boot) { |val| val.is_a?(Integer) && val >= 1 }
+                  optional(:multiplexing_scale_delay) { |val| val.is_a?(Integer) && val >= 1_000 }
+                end
+
+                # Makes sure min is not more than max
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next unless min(data)
+                  next unless max(data)
+
+                  min = min(data)
+                  max = max(data)
+
+                  next if min <= max
+
+                  [[%w[subscription_group_details], :multiplexing_min_max_mismatch]]
+                end
+
+                # Makes sure, that boot is between min and max
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next unless min(data)
+                  next unless max(data)
+                  next unless boot(data)
+
+                  min = min(data)
+                  max = max(data)
+                  boot = boot(data)
+
+                  next if boot.between?(min, max)
+
+                  [[%w[subscription_group_details], :multiplexing_boot_mismatch]]
+                end
+
+                # Makes sure, that boot is equal to min and max when not in dynamic mode
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next unless min(data)
+                  next unless max(data)
+                  next unless boot(data)
+
+                  min = min(data)
+                  max = max(data)
+                  boot = boot(data)
+
+                  # In dynamic mode there are other rules to check boot
+                  next if min != max
+                  next if boot == min
+
+                  [[%w[subscription_group_details], :multiplexing_boot_not_dynamic]]
+                end
+
+                # Makes sure we do not run multiplexing with 1 always which does not make much sense
+                # because then it behaves like without multiplexing and can create problems for
+                # users running multiplexed subscription groups with multiple topics
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next unless min(data)
+                  next unless max(data)
+
+                  min = min(data)
+                  max = max(data)
+
+                  next unless min == 1 && max == 1
+
+                  [[%w[subscription_group_details], :multiplexing_one_not_enough]]
+                end
+
+                class << self
+                  # @param data [Hash] topic details
+                  # @return [Integer, false] min or false if missing
+                  def min(data)
+                    data[:subscription_group_details].fetch(:multiplexing_min, false)
+                  end
+
+                  # @param data [Hash] topic details
+                  # @return [Integer, false] max or false if missing
+                  def max(data)
+                    data[:subscription_group_details].fetch(:multiplexing_max, false)
+                  end
+
+                  # @param data [Hash] topic details
+                  # @return [Integer, false] boot or false if missing
+                  def boot(data)
+                    data[:subscription_group_details].fetch(:multiplexing_boot, false)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{delaying/contracts/topic.rb → consumer_groups/multiplexing/patches/contracts/consumer_group.rb}

@@ -32,20 +32,25 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Delaying < Base
-          # Namespace for delaying feature contracts
-          module Contracts
-            # Contract to validate configuration of the expiring feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:delaying) do
-                required(:active) { |val| [true, false].include?(val) }
-                required(:delay) { |val| val.nil? || (val.is_a?(Integer) && val.positive?) }
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Patches to Karafka OSS
+            module Patches
+              # Contracts patches
+              module Contracts
+                # Consumer group contract patches
+                module ConsumerGroup
+                  # Redefines the setup allowing for multiple sgs as long as with different names
+                  #
+                  # @param topic [Hash] topic config hash
+                  # @return [Array] topic unique key for validators
+                  def topic_unique_key(topic)
+                    [
+                      topic[:name],
+                      topic[:subscription_group_details]
+                    ]
+                  end
+                end
               end
             end
           end

data/lib/karafka/pro/routing/features/{multiplexing/subscription_group.rb → consumer_groups/multiplexing/proxy.rb}

@@ -32,32 +32,26 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class Multiplexing < Base
-          # Adds methods needed for the multiplexing to work
-          module SubscriptionGroup
-            # @return [Config] multiplexing config
-            def multiplexing
-              @multiplexing ||= begin
-                max = @details.fetch(:multiplexing_max, 1)
-                min = @details.fetch(:multiplexing_min, max)
-                boot = @details.fetch(:multiplexing_boot, max / 2)
-                scale_delay = @details.fetch(:multiplexing_scale_delay, 60_000)
-                active = max > 1
-
-                Config.new(
-                  active: active,
-                  min: min,
-                  max: max,
-                  boot: boot,
-                  scale_delay: scale_delay
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Allows for multiplexing setup inside a consumer group definition
+            module Proxy
+              # @param min [Integer, nil] min multiplexing count or nil to set it to max, effectively
+              #   disabling dynamic multiplexing
+              # @param max [Integer] max multiplexing count
+              # @param boot [Integer] how many listeners should we start during boot by default
+              # @param scale_delay [Integer] number of ms of delay before applying any scale
+              #   operation to a consumer group
+              def multiplexing(min: nil, max: 1, boot: nil, scale_delay: 60_000)
+                @target.current_subscription_group_details.merge!(
+                  multiplexing_min: min || max,
+                  multiplexing_max: max,
+                  # Picks half of max by default as long as possible. Otherwise goes with min
+                  multiplexing_boot: boot || [min || max, (max / 2)].max,
+                  multiplexing_scale_delay: scale_delay
                 )
               end
             end
-
-            # @return [Boolean] is multiplexing active
-            def multiplexing?
-              multiplexing.active?
-            end
           end
         end
       end

data/lib/karafka/pro/routing/features/consumer_groups/multiplexing/subscription_group.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Multiplexing < Base
+            # Adds methods needed for the multiplexing to work
+            module SubscriptionGroup
+              # @return [Config] multiplexing config
+              def multiplexing
+                @multiplexing ||= begin
+                  max = @details.fetch(:multiplexing_max, 1)
+                  min = @details.fetch(:multiplexing_min, max)
+                  boot = @details.fetch(:multiplexing_boot, max / 2)
+                  scale_delay = @details.fetch(:multiplexing_scale_delay, 60_000)
+                  active = max > 1
+
+                  Config.new(
+                    active: active,
+                    min: min,
+                    max: max,
+                    boot: boot,
+                    scale_delay: scale_delay
+                  )
+                end
+              end
+
+              # @return [Boolean] is multiplexing active
+              def multiplexing?
+                multiplexing.active?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end