karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/direct_assignments/contracts/topic.rb

@@ -1,123 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class DirectAssignments < Base
-          # Namespace for direct assignments feature contracts
-          module Contracts
-            # Contract to validate configuration of the direct assignments topic feature
-            class Topic < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
-              end
-
-              nested(:direct_assignments) do
-                required(:active) { |val| [true, false].include?(val) }
-
-                required(:partitions) do |val|
-                  next true if val == true
-                  next false unless val.is_a?(Hash)
-                  next false unless val.keys.all?(Integer)
-                  next false unless val.values.all?(true)
-
-                  true
-                end
-              end
-
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                direct_assignments = data[:direct_assignments]
-                partitions = direct_assignments[:partitions]
-
-                next unless direct_assignments[:active]
-                next unless partitions.is_a?(Hash)
-                next unless partitions.empty?
-
-                [[%i[direct_assignments], :active_but_empty]]
-              end
-
-              # Make sure that when we use swarm, all requested partitions have allocation
-              # We cannot assign partitions and then not allocate them in swarm mode
-              # If this is the case, they should not be assigned in the first place
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                direct_assignments = data[:direct_assignments]
-                swarm = data[:swarm]
-
-                next unless direct_assignments[:active]
-                next unless swarm[:active]
-
-                nodes = swarm[:nodes]
-
-                next unless nodes.is_a?(Hash)
-                # Can be true for all partitions assignment and in this case we do not check
-                next unless direct_assignments[:partitions].is_a?(Hash)
-
-                direct_partitions = direct_assignments[:partitions].keys
-                swarm_partitions = nodes.values.flatten
-
-                next unless swarm_partitions.all?(Integer)
-                next if direct_partitions.sort == swarm_partitions.sort
-
-                # If we assigned more partitions than we distributed in swarm
-                if (direct_partitions - swarm_partitions).size.positive?
-                  [[%i[direct_assignments], :swarm_not_complete]]
-                else
-                  [[%i[direct_assignments], :swarm_overbooked]]
-                end
-              end
-
-              # Make sure that direct assignments are not used together with patterns as we
-              # cannot apply patterns on direct assignments
-              virtual do |data, errors|
-                next unless errors.empty?
-
-                direct_assignments = data[:direct_assignments]
-                patterns = data[:patterns]
-
-                next unless direct_assignments[:active]
-                next unless patterns[:active]
-
-                [[%i[direct_assignments], :patterns_active]]
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/direct_assignments/subscription_group.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Alterations to the direct assignments that allow us to do stable direct assignments
-        # without working with consumer groups dynamic assignments
-        class DirectAssignments < Base
-          # Extension allowing us to select correct subscriptions and assignments based on the
-          # expanded routing setup
-          module SubscriptionGroup
-            include Helpers::ConfigImporter.new(
-              swarm_node: %i[swarm node]
-            )
-
-            # @return [false, Array<String>] false if we do not have any subscriptions or array
-            #   with all the subscriptions for given subscription group
-            def subscriptions
-              topics
-                .select(&:active?)
-                .reject { |topic| topic.direct_assignments.active? }
-                .map(&:subscription_name)
-                .then { |subscriptions| subscriptions.empty? ? false : subscriptions }
-            end
-
-            # @param consumer [Karafka::Connection::Proxy] consumer for expanding  the partition
-            #   knowledge in case of certain topics assignments
-            # @return [Rdkafka::Consumer::TopicPartitionList] final tpl for assignments
-            def assignments(consumer)
-              topics
-                .select(&:active?)
-                .select { |topic| topic.direct_assignments.active? }
-                .to_h { |topic| build_assignments(topic) }
-                .tap { |topics| return false if topics.empty? }
-                .then { |topics| Iterator::Expander.new.call(topics) }
-                .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
-            end
-
-            private
-
-            # Builds assignments based on all the routing stuff. Takes swarm into consideration.
-            #
-            # @param topic [Karafka::Routing::Topic]
-            # @return [Array<String, Hash>]
-            def build_assignments(topic)
-              standard_setup = [
-                topic.subscription_name,
-                topic.direct_assignments.partitions
-              ]
-
-              return standard_setup unless swarm_node
-              # Unless user explicitly assigned particular partitions to particular nodes, we just
-              # go with full regular assignments
-              return standard_setup unless topic.swarm.nodes.is_a?(Hash)
-
-              [
-                topic.subscription_name,
-                topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
-              ]
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/direct_assignments/topic.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class DirectAssignments < Base
-          # Topic extensions for direct assignments
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @direct_assignments = nil
-            end
-
-            # Allows for direct assignment of
-            # @param partitions_or_all [true, Array<Integer>] informs Karafka that we want
-            #   to use direct assignments instead of automatic for this topic. It also allows us
-            #   to specify which partitions we're interested in or `true` if in all
-            #
-            # @example Assign all available partitions
-            #   direct_assignments(true)
-            #
-            # @example Assign partitions 2, 3 and 5
-            #   direct_assignments(2, 3, 5)
-            #
-            # @example Assign partitions from 0 to 3
-            #   direct_assignments(0..3)
-            def direct_assignments(*partitions_or_all)
-              @direct_assignments ||= if partitions_or_all == [true]
-                Config.new(
-                  active: true,
-                  partitions: true
-                )
-              elsif partitions_or_all.size == 1 && partitions_or_all.first.is_a?(Range)
-                partitions_or_all = partitions_or_all.first.to_a
-
-                Config.new(
-                  active: true,
-                  partitions: partitions_or_all.to_h { |partition| [partition, true] }
-                )
-              else
-                Config.new(
-                  active: !partitions_or_all.empty?,
-                  partitions: partitions_or_all.to_h { |partition| [partition, true] }
-                )
-              end
-            end
-
-            alias_method :assign, :direct_assignments
-
-            # @return [Hash] topic with all its native configuration options plus direct
-            #   assignments
-            def to_h
-              super.merge(
-                direct_assignments: direct_assignments.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/direct_assignments.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Allows for building direct assignments that bypass consumer group auto-assignments
-        # Useful for building complex pipelines that have explicit assignment requirements
-        class DirectAssignments < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/inline_insights/topic.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class InlineInsights < Base
-          # Routing topic inline insights API
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @inline_insights = nil
-            end
-
-            # @param active [Boolean] should inline insights be activated
-            # @param required [Boolean] are the insights required to operate
-            def inline_insights(active = -1, required: -1)
-              # This weird style of checking allows us to activate inline insights in few ways:
-              #   - inline_insights(true)
-              #   - inline_insights(required: true)
-              #   - inline_insights(required: false)
-              #
-              # In each of those cases inline insights will become active
-              @inline_insights ||= begin
-                config = Config.new(
-                  active: active == true || (active == -1 && required != -1),
-                  required: required == true
-                )
-
-                if config.active? && config.required?
-                  factory = lambda do |topic, partition|
-                    Pro::Processing::Filters::InlineInsightsDelayer.new(topic, partition)
-                  end
-
-                  filter(factory)
-                end
-
-                config
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/inline_insights.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Enhanced inline insights
-        # Allows you to set up an automatic filter that will ensure, that metrics are always
-        # available when processing starts.
-        class InlineInsights < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/long_running_job/config.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class LongRunningJob < Base
-          # Long-Running Jobs configuration
-          Config = Struct.new(
-            :active,
-            keyword_init: true
-          ) { alias_method :active?, :active }
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/long_running_job/topic.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class LongRunningJob < Base
-          # Long-Running Jobs topic API extensions
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @long_running_job = nil
-            end
-
-            # @param active [Boolean] do we want to enable long-running job feature for this topic
-            def long_running_job(active = false)
-              @long_running_job ||= Config.new(active: active)
-            end
-
-            alias_method :long_running, :long_running_job
-
-            # @return [Boolean] is a given job on a topic a long-running one
-            def long_running_job?
-              long_running_job.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus lrj
-            def to_h
-              super.merge(
-                long_running_job: long_running_job.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/multiplexing/contracts/routing.rb

@@ -1,81 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class Multiplexing < Base
-          module Contracts
-            # Contract that validates that dynamic multiplexing is not used with statistics
-            # disabled. Dynamic multiplexing relies on statistics emitted events for scaling
-            # decisions and will not function properly without them.
-            class Routing < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing")
-              end
-
-              # @param _builder [Karafka::Routing::Builder]
-              # @param scope [Array<String>]
-              def validate!(_builder, scope: [])
-                has_dynamic = Karafka::App
-                  .subscription_groups
-                  .values
-                  .flat_map(&:itself)
-                  .any? { |sg| sg.multiplexing? && sg.multiplexing.dynamic? }
-
-                return unless has_dynamic
-
-                stats_interval = App.config.kafka[:"statistics.interval.ms"]
-
-                return unless stats_interval
-                return unless stats_interval.zero?
-
-                super(
-                  { dynamic_multiplexing_with_statistics_disabled: true },
-                  scope: scope
-                )
-              end
-
-              virtual do |data, errors|
-                next unless errors.empty?
-                next unless data[:dynamic_multiplexing_with_statistics_disabled]
-
-                [[%i[multiplexing], :statistics_required_for_dynamic_multiplexing]]
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end