karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/consumer_groups/scheduled_messages/topic.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class ScheduledMessages < Base
+            # Topic extensions to be able to check if given topic is a scheduled messages topic
+            # Please note, that this applies to both the schedules topic and logs topic
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @scheduled_messages = nil
+                super
+              end
+
+              # @param active [Boolean] should this topic be considered related to scheduled messages
+              def scheduled_messages(active = false)
+                @scheduled_messages ||= Config.new(active: active)
+              end
+
+              # @return [Boolean] is this an ActiveJob topic
+              def scheduled_messages?
+                scheduled_messages.active?
+              end
+
+              # @return [Hash] topic with all its native configuration options plus scheduled
+              # messages namespace settings
+              def to_h
+                super.merge(
+                  scheduled_messages: scheduled_messages.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{recurring_tasks/proxy.rb → consumer_groups/scheduled_messages.rb}

@@ -32,10 +32,9 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class RecurringTasks < Base
-          # Routing proxy extensions for recurring tasks
-          module Proxy
-            include Builder
+        module ConsumerGroups
+          # Scheduled messages functionality routing namespace
+          class ScheduledMessages < Base
           end
         end
       end

data/lib/karafka/pro/routing/features/{non_blocking_job/topic.rb → consumer_groups/swarm/config.rb}

@@ -32,15 +32,16 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class NonBlockingJob < Base
-          # Non-Blocking Jobs topic API extensions
-          module Topic
-            # Non blocking setting method
-            def non_blocking_job(*)
-              long_running_job(*)
+        module ConsumerGroups
+          class Swarm < Base
+            # Swarm feature configuration
+            Config = Struct.new(
+              :active,
+              :nodes,
+              keyword_init: true
+            ) do
+              alias_method :active?, :active
             end
-
-            alias_method :non_blocking, :non_blocking_job
           end
         end
       end

data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/routing.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Swarm < Base
+            module Contracts
+              # Special contract that validates prior to starting swarm that each node has
+              # at least one assignment.
+              #
+              # It is special because we cannot run it during routing definitions, because we can
+              # only run it when all routes are defined and full context is available.
+              #
+              # This is why we use it before warmup when everything is expected to be configured.
+              class Routing < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing")
+                end
+
+                # Validates that each node has at least one assignment.
+                #
+                # @param builder [Karafka::Routing::Builder]
+                # @param scope [Array<String>]
+                def validate!(builder, scope: [])
+                  nodes_setup = Hash.new do |h, node_id|
+                    h[node_id] = { active: false, node_id: node_id }
+                  end
+
+                  # Initialize nodes in the hash so we can iterate over them
+                  App.config.swarm.nodes.times { |node_id| nodes_setup[node_id] }
+                  nodes_setup.freeze
+
+                  builder.each do |group|
+                    group.topics.each do |topic|
+                      nodes_setup.each do |node_id, details|
+                        next unless topic.active?
+                        next unless topic.swarm.nodes.include?(node_id)
+
+                        details[:active] = true
+                      end
+                    end
+                  end
+
+                  nodes_setup.each_value do |details|
+                    super(details, scope: scope)
+                  end
+                end
+
+                virtual do |data, errors|
+                  next unless errors.empty?
+                  next if data[:active]
+
+                  [[%i[swarm_nodes], :not_used]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/swarm/contracts/topic.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Swarm < Base
+            # Namespace for swarm contracts
+            module Contracts
+              # Contract to validate configuration of the swarm feature
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
+
+                nested(:swarm) do
+                  required(:active) { |val| val == true }
+
+                  required(:nodes) do |val|
+                    next true if val.is_a?(Range)
+                    next true if val.is_a?(Array) && val.all?(Integer)
+                    next false unless val.is_a?(Hash)
+                    next false unless val.keys.all?(Integer)
+
+                    values = val.values
+
+                    next false unless values.all? { |ps| ps.is_a?(Array) || ps.is_a?(Range) }
+                    next true if values.flatten.all?(Integer)
+                    next true if values.flatten.all?(Range)
+
+                    false
+                  end
+                end
+
+                # Make sure that if range is defined it fits number of nodes (except infinity)
+                # As it may be a common error to accidentally define a node that will never be
+                # reached
+                virtual do |data, errors|
+                  next unless errors.empty?
+
+                  nodes = data[:swarm][:nodes]
+                  # Hash can be used when we use direct assignments. In such cases nodes ids are
+                  # keys in the hash and values are partitions
+                  nodes = nodes.keys if nodes.is_a?(Hash)
+
+                  # Defaults
+                  next if nodes.first.zero? && nodes.include?(Float::INFINITY)
+
+                  # If our expectation towards which node should run things matches at least one
+                  # node, then it's ok
+                  next if Karafka::App.config.swarm.nodes.times.any? do |node_id|
+                    nodes.include?(node_id)
+                  end
+
+                  [[%i[swarm_nodes], :with_non_existent_nodes]]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/swarm/topic.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class Swarm < Base
+            # Topic swarm API extensions
+            module Topic
+              # This method sets up the extra instance variable to nil before calling
+              # the parent class initializer. The explicit initialization
+              # to nil is included as an optimization for Ruby's object shapes system,
+              # which improves memory layout and access performance.
+              def initialize(...)
+                @swarm = nil
+                super
+              end
+
+              # Allows defining swarm routing topic settings
+              # @param nodes [Range, Array, Hash] range of nodes ids or array with nodes ids for
+              #   which we should run given topic or hash with nodes expected partition assignments
+              #   for the direct assignments API.
+              #
+              # @example Assign given topic only to node 1
+              #   swarm(nodes: [1])
+              #
+              # @example Assign given topic to nodes from 1 to 3
+              #   swarm(nodes: 1..3)
+              #
+              # @example Assign partitions 2 and 3 to node 0 and partitions 0, 1 to node 1
+              #   swarm(
+              #     nodes: {
+              #       0 => [2, 3],
+              #       1 => [0, 1]
+              #     }
+              #   )
+              #
+              # @example Assign partitions in ranges to nodes
+              #   swarm(
+              #     nodes: {
+              #       0 => (0..2),
+              #       1 => (3..5)
+              #     }
+              #   )
+              def swarm(nodes: (0...Karafka::App.config.swarm.nodes))
+                @swarm ||= Config.new(active: true, nodes: nodes)
+              end
+
+              # @return [true] swarm setup is always true. May not be in use but is active
+              def swarm?
+                swarm.active?
+              end
+
+              # @return [Boolean] should this topic be active. In the context of swarm it is only
+              #   active when swarm routing setup does not limit nodes on which it should operate
+              def active?
+                node = Karafka::App.config.swarm.node
+
+                return super unless node
+
+                super && swarm.nodes.include?(node.id)
+              end
+
+              # @return [Hash] topic with all its native configuration options plus swarm
+              def to_h
+                super.merge(
+                  swarm: swarm.to_h
+                ).freeze
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/swarm.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          # Karafka Pro Swarm extensions to the routing
+          # They allow for more granular work assignment in the swarm
+          class Swarm < Base
+            class << self
+              # Binds our routing validation contract prior to warmup in the supervisor, so we can
+              # run it when all the context should be there (config + full routing)
+              #
+              # @param config [Karafka::Core::Configurable::Node]
+              def post_setup(config)
+                config.monitor.subscribe("app.before_warmup") do
+                  Contracts::Routing.new.validate!(
+                    config.internal.routing.builder,
+                    scope: %w[swarm]
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/consumer_groups/virtual_partitions/config.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        module ConsumerGroups
+          class VirtualPartitions < Base
+            # Configuration for virtual partitions feature
+            Config = Struct.new(
+              :active,
+              :partitioner,
+              :max_partitions,
+              :offset_metadata_strategy,
+              :reducer,
+              :distribution,
+              keyword_init: true
+            ) do
+              # @return [Boolean] is this feature active
+              def active?
+                active
+              end
+
+              # @return [Object] distributor instance for the current distribution
+              def distributor
+                @distributor ||= case distribution
+                when :balanced
+                  Processing::ConsumerGroups::VirtualPartitions::Distributors::Balanced.new(self)
+                when :consistent
+                  Processing::ConsumerGroups::VirtualPartitions::Distributors::Consistent.new(self)
+                else
+                  raise Karafka::Errors::UnsupportedCaseError, distribution
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/{parallel_segments/contracts/consumer_group.rb → consumer_groups/virtual_partitions/contracts/topic.rb}

@@ -32,35 +32,38 @@ module Karafka
   module Pro
     module Routing
       module Features
-        class ParallelSegments < Base
-          # Namespace for parallel segments contracts
-          module Contracts
-            # Contract to validate configuration of the parallel segments feature
-            class ConsumerGroup < Karafka::Contracts::Base
-              configure do |config|
-                config.error_messages = YAML.safe_load_file(
-                  File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
-                ).fetch("en").fetch("validations").fetch("routing").fetch("consumer_group")
+        module ConsumerGroups
+          class VirtualPartitions < Base
+            # Namespace for VP contracts
+            module Contracts
+              # Rules around virtual partitions
+              class Topic < Karafka::Contracts::Base
+                configure do |config|
+                  config.error_messages = YAML.safe_load_file(
+                    File.join(Karafka.gem_root, "config", "locales", "pro_errors.yml")
+                  ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+                end
 
-                nested(:parallel_segments) do
+                nested(:virtual_partitions) do
                   required(:active) { |val| [true, false].include?(val) }
                   required(:partitioner) { |val| val.nil? || val.respond_to?(:call) }
                   required(:reducer) { |val| val.respond_to?(:call) }
-                  required(:count) { |val| val.is_a?(Integer) && val >= 1 }
-                  required(:merge_key) { |val| val.is_a?(String) && val.size >= 1 }
+                  required(:max_partitions) { |val| val.is_a?(Integer) && val >= 1 }
+                  required(:offset_metadata_strategy) { |val| %i[exact current].include?(val) }
+                  required(:distribution) { |val| %i[consistent balanced].include?(val) }
                 end
 
-                # When parallel segments are defined, partitioner needs to respond to `#call` and
-                # it cannot be nil
+                # When virtual partitions are defined, partitioner needs to respond to `#call` and it
+                # cannot be nil
                 virtual do |data, errors|
                   next unless errors.empty?
 
-                  parallel_segments = data[:parallel_segments]
+                  virtual_partitions = data[:virtual_partitions]
 
-                  next unless parallel_segments[:active]
-                  next if parallel_segments[:partitioner].respond_to?(:call)
+                  next unless virtual_partitions[:active]
+                  next if virtual_partitions[:partitioner].respond_to?(:call)
 
-                  [[%i[parallel_segments partitioner], :respond_to_call]]
+                  [[%i[virtual_partitions partitioner], :respond_to_call]]
                 end
               end
             end