karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/config.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class DirectAssignments < Base
+            # Direct assignments feature configuration
+            Config = Struct.new(:active, :partitions, keyword_init: true) do
+              alias_method :active?, :active
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/consumer_group.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class DirectAssignments < Base
+            module Contracts
+              # Contract to validate configuration of the direct assignments feature
+              class ConsumerGroup < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
+
+                  virtual do |data, errors|
+                    next unless errors.empty?
+
+                    active = data[:topics].map do |topic|
+                      topic[:direct_assignments][:active]
+                    end
+
+                    # If none active we use standard subscriptions
+                    next if active.none?
+                    # If all topics from a given CG are using direct assignments that is expected
+                    # and allowed
+                    next if active.all?
+
+                    [[%i[direct_assignments], :homogenous]]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/contracts/topic.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class DirectAssignments < Base
+            # Namespace for direct assignments feature contracts
+            module Contracts
+              # Contract to validate configuration of the direct assignments topic feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:direct_assignments) do
+                  required(:active) { |val| [true, false].include?(val) }
+
+                  required(:partitions) do |val|
+                    next true if val == true
+                    next false unless val.is_a?(Hash)
+                    next false unless val.keys.all?(Integer)
+                    next false unless val.values.all?(true)
+
+                    true
+                  end
+                end
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  direct_assignments = data[:direct_assignments]
+                  partitions = direct_assignments[:partitions]
+
+                  next unless direct_assignments[:active]
+                  next unless partitions.is_a?(Hash)
+                  next unless partitions.empty?
+
+                  [[%i[direct_assignments], :active_but_empty]]
+                end
+
+                # Make sure that when we use swarm, all requested partitions have allocation
+                # We cannot assign partitions and then not allocate them in swarm mode
+                # If this is the case, they should not be assigned in the first place
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  direct_assignments = data[:direct_assignments]
+                  swarm = data[:swarm]
+
+                  next unless direct_assignments[:active]
+                  next unless swarm[:active]
+
+                  nodes = swarm[:nodes]
+
+                  next unless nodes.is_a?(Hash)
+                  # Can be true for all partitions assignment and in this case we do not check
+                  next unless direct_assignments[:partitions].is_a?(Hash)
+
+                  direct_partitions = direct_assignments[:partitions].keys
+                  swarm_partitions = nodes.values.flatten
+
+                  next unless swarm_partitions.all?(Integer)
+                  next if direct_partitions.sort == swarm_partitions.sort
+
+                  # If we assigned more partitions than we distributed in swarm
+                  if (direct_partitions - swarm_partitions).size.positive?
+                    [[%i[direct_assignments], :swarm_not_complete]]
+                  else
+                    [[%i[direct_assignments], :swarm_overbooked]]
+                  end
+                end
+
+                # Make sure that direct assignments are not used together with patterns as we
+                # cannot apply patterns on direct assignments
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  direct_assignments = data[:direct_assignments]
+                  patterns = data[:patterns]
+
+                  next unless direct_assignments[:active]
+                  next unless patterns[:active]
+
+                  [[%i[direct_assignments], :patterns_active]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/subscription_group.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Alterations to the direct assignments that allow us to do stable direct assignments
+          # without working with consumer groups dynamic assignments
+          class DirectAssignments < Base
+            # Extension allowing us to select correct subscriptions and assignments based on the
+            # expanded routing setup
+            module SubscriptionGroup
+              include Helpers::ConfigImporter.new(
+                swarm_node: %i[swarm node]
+              )
+
+              # @return [false, Array<String>] false if we do not have any subscriptions or array
+              #   with all the subscriptions for given subscription group
+              def subscriptions
+                topics
+                  .select(&:active?)
+                  .reject { |topic| topic.direct_assignments.active? }
+                  .map(&:subscription_name)
+                  .then { |subscriptions| subscriptions.empty? ? false : subscriptions }
+              end
+
+              # @param consumer [Karafka::Connection::Proxy] consumer for expanding  the partition
+              #   knowledge in case of certain topics assignments
+              # @return [Rdkafka::Consumer::TopicPartitionList] final tpl for assignments
+              def assignments(consumer)
+                topics
+                  .select(&:active?)
+                  .select { |topic| topic.direct_assignments.active? }
+                  .to_h { |topic| build_assignments(topic) }
+                  .tap { |topics| return false if topics.empty? }
+                  .then { |topics| Iterator::Expander.new.call(topics) }
+                  .then { |topics| Iterator::TplBuilder.new(consumer, topics).call }
+              end
+
+              private
+
+              # Builds assignments based on all the routing stuff. Takes swarm into consideration.
+              #
+              # @param topic [Karafka::Routing::Topic]
+              # @return [Array<String, Hash>]
+              def build_assignments(topic)
+                standard_setup = [
+                  topic.subscription_name,
+                  topic.direct_assignments.partitions
+                ]
+
+                return standard_setup unless swarm_node
+                # Unless user explicitly assigned particular partitions to particular nodes, we just
+                # go with full regular assignments
+                return standard_setup unless topic.swarm.nodes.is_a?(Hash)
+
+                [
+                  topic.subscription_name,
+                  topic.swarm.nodes.fetch(swarm_node.id).to_h { |partition| [partition, true] }
+                ]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments/topic.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class DirectAssignments < Base
+            # Topic extensions for direct assignments
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @direct_assignments = nil
+                super
+              end
+
+              # Allows for direct assignment of
+              # @param partitions_or_all [true, Array<Integer>] informs Karafka that we want
+              #   to use direct assignments instead of automatic for this topic. It also allows us
+              #   to specify which partitions we're interested in or `true` if in all
+              #
+              # @example Assign all available partitions
+              #   direct_assignments(true)
+              #
+              # @example Assign partitions 2, 3 and 5
+              #   direct_assignments(2, 3, 5)
+              #
+              # @example Assign partitions from 0 to 3
+              #   direct_assignments(0..3)
+              def direct_assignments(*partitions_or_all)
+                @direct_assignments ||= if partitions_or_all == [true]
+                  Config.new(
+                    active: true,
+                    partitions: true
+                  )
+                elsif partitions_or_all.size == 1 && partitions_or_all.first.is_a?(Range)
+                  partitions_or_all = partitions_or_all.first.to_a
+
+                  Config.new(
+                    active: true,
+                    partitions: partitions_or_all.to_h { |partition| [partition, true] }
+                  )
+                else
+                  Config.new(
+                    active: !partitions_or_all.empty?,
+                    partitions: partitions_or_all.to_h { |partition| [partition, true] }
+                  )
+                end
+              end
+
+              alias_method :assign, :direct_assignments
+
+              # @return [Hash] topic with all its native configuration options plus direct
+              #   assignments
+              def to_h
+                super.merge(
+                  direct_assignments: direct_assignments.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/direct_assignments.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Allows for building direct assignments that bypass consumer group auto-assignments
+          # Useful for building complex pipelines that have explicit assignment requirements
+          class DirectAssignments < Base
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class InlineInsights < Base
+            # Config of this feature
+            Config = Struct.new(
+              :active,
+              :required,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
+              alias_method :required?, :required
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/contracts/topic.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class InlineInsights < Base
+            # Inline Insights related contracts namespace
+            module Contracts
+              # Contract for inline insights topic setup
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested :inline_insights do
+                  required(:active) { |val| [true, false].include?(val) }
+                  required(:required) { |val| [true, false].include?(val) }
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/inline_insights/topic.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class InlineInsights < Base
+            # Routing topic inline insights API
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @inline_insights = nil
+                super
+              end
+
+              # @param active [Boolean] should inline insights be activated
+              # @param required [Boolean] are the insights required to operate
+              def inline_insights(active = -1, required: -1)
+                # This weird style of checking allows us to activate inline insights in few ways:
+                #   - inline_insights(true)
+                #   - inline_insights(required: true)
+                #   - inline_insights(required: false)
+                #
+                # In each of those cases inline insights will become active
+                @inline_insights ||= begin
+                  config = Config.new(
+                    active: active == true || (active == -1 && required != -1),
+                    required: required == true
+                  )
+
+                  if config.active? && config.required?
+                    factory = lambda do |topic, partition|
+                      Pro::Processing::ConsumerGroups::Filters::InlineInsightsDelayer.new(topic, partition)
+                    end
+
+                    filter(factory)
+                  end
+
+                  config
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end