karafka-web 0.11.5 → 0.11.6

data/lib/karafka/web/pro/commanding/handlers/partitions/listener.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Web
@@ -10,7 +27,7 @@ module Karafka
         module Handlers
           module Partitions
             # Listener that hooks to the connection listener fetch loop flow to adjust it prior
-            # to the next pooling or on rebalances to execute the partition specific commands.
+            # to the next polling or on rebalances to execute the partition specific commands.
             class Listener
               def initialize
                 @tracker = Tracker.instance
@@ -18,15 +35,27 @@ module Karafka
               end
 
               # Connects to the fetching loop pre-fetch and executes requested commands before
-              # pooling (if any).
+              # polling (if any).
               #
               # @param event [Karafka::Core::Monitoring::Event]
               def on_connection_listener_fetch_loop(event)
                 listener = event[:caller]
                 client = event[:client]
+                subscription_group = listener.subscription_group
+                consumer_group_id = subscription_group.consumer_group.id
 
-                @tracker.each_for(listener.subscription_group.id) do |command|
-                  @executor.call(listener, client, command)
+                # Get current partition assignments from the client
+                assignments = client.assignment.to_h
+
+                # Iterate over all assigned topics and partitions
+                assignments.each do |topic_name, partitions|
+                  partitions.each do |partition|
+                    partition_id = partition.partition
+
+                    @tracker.each_for(consumer_group_id, topic_name, partition_id) do |command|
+                      @executor.call(listener, client, command)
+                    end
+                  end
                 end
               end
 
@@ -36,8 +65,18 @@ module Karafka
               # Rejects all the commands if there were any waiting.
               # @param event [Karafka::Core::Monitoring::Event]
               def on_rebalance_partitions_assigned(event)
-                @tracker.each_for(event[:subscription_group_id]) do |command|
-                  @executor.reject(command)
+                subscription_group = event[:subscription_group]
+                consumer_group_id = subscription_group.consumer_group.id
+
+                # On rebalance, reject any pending commands for all topics and partitions
+                # that were assigned to this subscription group
+                subscription_group.topics.each do |topic|
+                  # Only iterate over partitions that actually have pending commands
+                  @tracker.partition_ids_for(consumer_group_id, topic.name).each do |partition_id|
+                    @tracker.each_for(consumer_group_id, topic.name, partition_id) do |command|
+                      @executor.reject(command)
+                    end
+                  end
                 end
               end
 

data/lib/karafka/web/pro/commanding/handlers/partitions/tracker.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
 
-# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
-# See LICENSE for details.
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
 
 module Karafka
   module Web
@@ -22,36 +39,68 @@ module Karafka
               def initialize
                 @mutex = Mutex.new
                 @requests = Hash.new { |h, k| h[k] = [] }
+                # Index tracking which partitions have pending commands per consumer_group:topic
+                @partition_index = Hash.new { |h, k| h[k] = Set.new }
               end
 
               # Adds the given command into the tracker so it can be retrieved when needed.
               #
               # @param command [Request] command we want to schedule
-              # @note We accumulate requests per subscription group because this is the layer of
-              #   applicability of those even for partition related requests.
+              # @note Commands are indexed by consumer_group_id:topic:partition_id combination since
+              #   partition commands are dispatched without subscription_group_id.
               def <<(command)
+                key = "#{command[:consumer_group_id]}:#{command[:topic]}:#{command[:partition_id]}"
+                index_key = "#{command[:consumer_group_id]}:#{command[:topic]}"
+
                 @mutex.synchronize do
-                  @requests[command[:subscription_group_id]] << command
+                  @requests[key] << command
+                  @partition_index[index_key] << command[:partition_id]
                 end
               end
 
-              # Selects all incoming command requests for given subscription group and iterates
-              # over them. It removes selected requests during iteration.
+              # Selects all incoming command requests for given consumer group, topic, and partition
+              # and iterates over them. It removes selected requests during iteration.
               #
-              # @param subscription_group_id [String] id of the subscription group for which we
-              #   want to get all the requests. Subscription groups ids (not names) are unique
-              #   within the application, so it is unique "enough".
+              # @param consumer_group_id [String]
+              # @param topic [String]
+              # @param partition_id [Integer]
               #
-              # @yieldparam [Request] given command request for the requested subscription group
-              def each_for(subscription_group_id, &)
+              # @yieldparam [Request] given command request
+              def each_for(consumer_group_id, topic, partition_id, &)
+                key = "#{consumer_group_id}:#{topic}:#{partition_id}"
+                index_key = "#{consumer_group_id}:#{topic}"
                 requests = nil
 
                 @mutex.synchronize do
-                  requests = @requests.delete(subscription_group_id)
+                  requests = @requests.delete(key)
+                  @partition_index[index_key].delete(partition_id) if requests
                 end
 
                 (requests || EMPTY_ARRAY).each(&)
               end
+
+              # Returns partition IDs that have pending commands for the given consumer group
+              # and topic
+              #
+              # @param consumer_group_id [String]
+              # @param topic [String]
+              # @return [Array<Integer>] partition IDs with pending commands
+              def partition_ids_for(consumer_group_id, topic)
+                index_key = "#{consumer_group_id}:#{topic}"
+
+                @mutex.synchronize do
+                  @partition_index[index_key].to_a
+                end
+              end
+
+              # Clears all stored requests and partition index
+              # @note Primarily for testing purposes
+              def clear!
+                @mutex.synchronize do
+                  @requests.clear
+                  @partition_index.clear
+                end
+              end
             end
           end
         end

data/lib/karafka/web/pro/commanding/handlers/topics/commands/base.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Topics
+            # Namespace for topic related commands
+            module Commands
+              # Base class for all the topic related commands handlers
+              class Base
+                # @param listener [Karafka::Connection::Listener] listener that handles given
+                #   topic in the context of given subscription group
+                # @param client [Karafka::Connection::Client] underlying Kafka client
+                # @param request [Request] command request
+                def initialize(listener, client, request)
+                  @listener = listener
+                  @client = client
+                  @request = request
+                end
+
+                # Runs the command
+                def call
+                  raise NotImplementedError, "Implement in a subclass"
+                end
+
+                private
+
+                attr_reader :listener, :client, :request
+
+                # @return [String] name of the topic on which the command should be applied
+                def topic
+                  @topic ||= request[:topic]
+                end
+
+                # Finds all partition IDs for the target topic that are currently assigned
+                # to this listener's subscription group.
+                #
+                # @return [Array<Integer>] array of partition IDs assigned to this process for
+                #   the target topic
+                def owned_partition_ids
+                  @owned_partition_ids ||= begin
+                    assignment = client.assignment
+
+                    # assignment is a TopicPartitionList from rdkafka
+                    # We need to find partitions that match our target topic
+                    assignment.to_h.fetch(topic, []).map(&:partition)
+                  end
+                end
+
+                # Gets or creates a coordinator for a given partition
+                #
+                # @param partition_id [Integer] partition ID
+                # @return [Karafka::Processing::Coordinator, Karafka::Pro::Processing::Coordinator]
+                def coordinator_for(partition_id)
+                  listener.coordinators.find_or_create(topic, partition_id)
+                end
+
+                # @return [String] id of the current consumer process
+                def process_id
+                  @process_id ||= ::Karafka::Web.config.tracking.consumers.sampler.process_id
+                end
+
+                # Publishes the execution result back to Kafka
+                #
+                # @param status [String] execution status
+                # @param partitions_affected [Array<Integer>] partition IDs that were affected
+                # @param partitions_prevented [Array<Integer>] partition IDs that were skipped
+                #   due to prevent_override setting
+                def result(status, partitions_affected: [], partitions_prevented: [])
+                  Commanding::Dispatcher.result(
+                    request.name,
+                    process_id,
+                    request.to_h.merge(
+                      status: status,
+                      partitions_affected: partitions_affected,
+                      partitions_prevented: partitions_prevented
+                    )
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/topics/commands/pause.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Topics
+            module Commands
+              # Executes the topic-level pause request to pause all partitions of a topic
+              # that are assigned to this consumer process within the target consumer group.
+              class Pause < Base
+                # 10 years in ms - effectively forever
+                FOREVER_MS = 10 * 365 * 24 * 60 * 60 * 1000
+
+                private_constant :FOREVER_MS
+
+                # Triggers pausing of all partitions for the target topic that are owned by
+                # this process. Supports prevent_override to skip already paused partitions.
+                def call
+                  partitions_affected = []
+                  partitions_prevented = []
+
+                  duration = request[:duration]
+                  duration = FOREVER_MS if duration.zero?
+                  prevent_override = request[:prevent_override]
+
+                  owned_partition_ids.each do |partition_id|
+                    coordinator = coordinator_for(partition_id)
+
+                    # If prevent_override is set and partition is already paused, skip it
+                    if coordinator.pause_tracker.paused? && prevent_override
+                      partitions_prevented << partition_id
+                      next
+                    end
+
+                    coordinator.pause_tracker.pause(duration)
+                    client.pause(topic, partition_id, nil, duration)
+
+                    partitions_affected << partition_id
+                  end
+
+                  result(
+                    "applied",
+                    partitions_affected: partitions_affected,
+                    partitions_prevented: partitions_prevented
+                  )
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/topics/commands/resume.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Topics
+            module Commands
+              # Resumes all paused partitions for a topic that are assigned to this consumer process
+              # within the target consumer group.
+              class Resume < Base
+                # Expires the pause on all partitions for the target topic so Karafka resumes
+                # processing of those partitions.
+                def call
+                  partitions_affected = []
+
+                  owned_partition_ids.each do |partition_id|
+                    coordinator = coordinator_for(partition_id)
+
+                    coordinator.pause_tracker.expire
+                    coordinator.pause_tracker.reset if request[:reset_attempts]
+
+                    partitions_affected << partition_id
+                  end
+
+                  result("applied", partitions_affected: partitions_affected)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/topics/executor.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Topics
+            # Selects proper command for running prior to next poll and executes its flow
+            class Executor
+              # @param listener [Karafka::Connection::Listener]
+              # @param client [Karafka::Connection::Client]
+              # @param request [Request]
+              def call(listener, client, request)
+                command = case request.name
+                when Commanding::Commands::Topics::Resume.name
+                  Commands::Resume
+                when Commanding::Commands::Topics::Pause.name
+                  Commands::Pause
+                else
+                  raise ::Karafka::Errors::UnsupportedCaseError, request.name
+                end
+
+                command.new(
+                  listener,
+                  client,
+                  request
+                ).call
+              end
+
+              # Publishes the reject event as the final result. Used to indicate, that given
+              # request will not be processed because it is not valid anymore.
+              #
+              # @param request [Request]
+              def reject(request)
+                Dispatcher.result(
+                  request.name,
+                  process_id,
+                  request.to_h.merge(status: "rebalance_rejected")
+                )
+              end
+
+              private
+
+              # @return [String] id of the current consumer process
+              def process_id
+                ::Karafka::Web.config.tracking.consumers.sampler.process_id
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/topics/listener.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          module Topics
+            # Listener that hooks to the connection listener fetch loop flow to adjust it prior
+            # to the next polling or on rebalances to execute the topic specific commands.
+            class Listener
+              def initialize
+                @tracker = Tracker.instance
+                @executor = Executor.new
+              end
+
+              # Connects to the fetching loop pre-fetch and executes requested commands before
+              # polling (if any).
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_connection_listener_fetch_loop(event)
+                listener = event[:caller]
+                client = event[:client]
+                subscription_group = listener.subscription_group
+                consumer_group_id = subscription_group.consumer_group.id
+
+                # Iterate over all topics in this subscription group and check for commands
+                subscription_group.topics.each do |topic|
+                  @tracker.each_for(consumer_group_id, topic.name) do |command|
+                    @executor.call(listener, client, command)
+                  end
+                end
+              end
+
+              # Creates a rebalance barrier, so we do not execute any commands in between
+              # rebalances. This prevents us from aggregating old and outdated requests.
+              #
+              # Rejects all the commands if there were any waiting.
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_rebalance_partitions_assigned(event)
+                subscription_group = event[:subscription_group]
+                consumer_group_id = subscription_group.consumer_group.id
+
+                subscription_group.topics.each do |topic|
+                  @tracker.each_for(consumer_group_id, topic.name) do |command|
+                    @executor.reject(command)
+                  end
+                end
+              end
+
+              # @param event [Karafka::Core::Monitoring::Event]
+              # @see `#on_rebalance_partitions_assigned`
+              def on_rebalance_partitions_revoked(event)
+                on_rebalance_partitions_assigned(event)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/pro/commanding/handlers/topics/tracker.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Web
+    module Pro
+      module Commanding
+        module Handlers
+          # Namespace for handling requests related to topic-level operations
+          module Topics
+            # Tracker used to record incoming topic related operational requests until they are
+            # executable or invalid. It stores the requests as they come for execution pre-polling.
+            class Tracker
+              include Singleton
+
+              # Empty array for internal usage
+              EMPTY_ARRAY = [].freeze
+
+              private_constant :EMPTY_ARRAY
+
+              def initialize
+                @mutex = Mutex.new
+                @requests = Hash.new { |h, k| h[k] = [] }
+              end
+
+              # Adds the given command into the tracker so it can be retrieved when needed.
+              #
+              # @param command [Request] command we want to schedule
+              # @note Commands are indexed by consumer_group_id:topic combination since topic
+              #   commands are dispatched without subscription_group_id.
+              def <<(command)
+                key = "#{command[:consumer_group_id]}:#{command[:topic]}"
+
+                @mutex.synchronize do
+                  @requests[key] << command
+                end
+              end
+
+              # Selects all incoming command requests that match the given consumer group and topic
+              # and iterates over them. It removes selected requests during iteration.
+              #
+              # @param consumer_group_id [String]
+              # @param topic [String]
+              #
+              # @yieldparam [Request] given command request
+              def each_for(consumer_group_id, topic, &)
+                key = "#{consumer_group_id}:#{topic}"
+                requests = nil
+
+                @mutex.synchronize do
+                  requests = @requests.delete(key)
+                end
+
+                (requests || EMPTY_ARRAY).each(&)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end