karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/declaratives/builder.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Declaratives
+    # The main entry point for the declaratives subsystem. Returned by `Karafka::App.declaratives`.
+    # Provides the `draw` DSL and programmatic access to topic declarations and operations.
+    class Builder
+      attr_reader :repository
+
+      def initialize
+        @repository = Repository.new
+        @defaults = ->(_) {}
+      end
+
+      # DSL entry point for declaring topics outside of routing blocks.
+      #
+      # @param block [Proc] block to evaluate in builder context
+      #
+      # @example
+      #   Karafka::App.declaratives.draw do
+      #     topic :orders do
+      #       partitions 10
+      #       replication_factor 3
+      #       config 'retention.ms' => 604_800_000
+      #     end
+      #   end
+      def draw(&block)
+        instance_eval(&block)
+      end
+
+      # Sets default values applied to every topic declared after this call.
+      # Defaults are evaluated before the topic block, so topic-specific values override them.
+      #
+      # @param block [Proc] block evaluated in each topic's context before the topic block
+      def defaults(&block)
+        @defaults = block
+      end
+
+      # Declares a topic with the given name. If a topic with this name was already declared
+      # (via routing bridge or a prior draw call), the existing declaration is returned and
+      # the block is evaluated on it (allowing additive configuration).
+      #
+      # @param name [String, Symbol] topic name
+      # @param block [Proc] optional block evaluated in the topic's context
+      # @return [Karafka::Declaratives::Topic] the declaration
+      def topic(name, &block)
+        declaration = @repository.find_or_create(name)
+        declaration.instance_eval(&@defaults)
+        declaration.instance_eval(&block) if block
+        declaration
+      end
+
+      # @return [Array<Karafka::Declaratives::Topic>] all active topic declarations
+      def topics
+        @repository.active
+      end
+
+      # @param name [String, Symbol] topic name
+      # @return [Karafka::Declaratives::Topic, nil] specific topic declaration
+      def find_topic(name)
+        @repository.find(name)
+      end
+    end
+  end
+end

data/lib/karafka/declaratives/contracts/topic.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Declaratives
+    # Namespace for declaratives-specific validation contracts
+    module Contracts
+      # Validates declarative topic configuration. This is the canonical location for declarative
+      # topic validation, replacing the routing feature contract.
+      class Topic < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+          ).fetch("en").fetch("validations").fetch("declaratives").fetch("topic")
+        end
+
+        nested :declaratives do
+          required(:active) { |val| [true, false].include?(val) }
+          required(:partitions) { |val| val.is_a?(Integer) && val.positive? }
+          required(:replication_factor) { |val| val.is_a?(Integer) && val.positive? }
+          required(:details) do |val|
+            val.is_a?(Hash) &&
+              val.keys.all?(Symbol)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/declaratives/repository.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Declaratives
+    # Holds all topic declarations. Single source of truth for "what topics have been declared."
+    # Both the new DSL and the routing bridge populate this repository.
+    class Repository
+      def initialize
+        @topics = {}
+      end
+
+      # Finds an existing declaration or creates a new one for the given topic name
+      # @param name [String, Symbol] topic name
+      # @return [Karafka::Declaratives::Topic] the declaration
+      def find_or_create(name)
+        @topics[name.to_s] ||= Topic.new(name)
+      end
+
+      # @return [Array<Karafka::Declaratives::Topic>] all declarations where active? is true
+      def active
+        @topics.values.select(&:active?)
+      end
+
+      # @param name [String, Symbol] topic name
+      # @return [Karafka::Declaratives::Topic, nil] the declaration or nil
+      def find(name)
+        @topics[name.to_s]
+      end
+
+      # Iterates over all declarations
+      # @param block [Proc] block to yield each topic to
+      def each(&block)
+        @topics.each_value(&block)
+      end
+
+      # @return [Integer] number of declarations
+      def size
+        @topics.size
+      end
+
+      # @return [Boolean] are there any declarations
+      def empty?
+        @topics.empty?
+      end
+
+      # Removes all declarations
+      def clear
+        @topics.clear
+      end
+    end
+  end
+end

data/lib/karafka/declaratives/topic.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Declaratives
+    # Represents a single declarative topic definition - what a topic should look like on the
+    # broker. This is a standalone object, independent of routing concepts like consumers or
+    # subscription groups.
+    class Topic
+      attr_reader :name, :details
+      attr_writer :active, :partitions, :replication_factor, :details
+
+      # Bootstrap servers for the Kafka cluster this topic belongs to.
+      # Set by the routing bridge from the routing topic's kafka config.
+      # Topics declared via the standalone DSL leave this nil (treated as default cluster).
+      attr_accessor :bootstrap_servers
+
+      # @param name [String, Symbol] topic name
+      def initialize(name)
+        @name = name.to_s
+        @active = true
+        @partitions = 1
+        @replication_factor = 1
+        @details = {}
+        @bootstrap_servers = nil
+      end
+
+      # Self-accessor for backwards compatibility.
+      # CLI commands that previously accessed topic.declaratives.partitions on routing topics
+      # can now call the same on Declaratives::Topic instances directly.
+      #
+      # @return [Karafka::Declaratives::Topic] self
+      def declaratives
+        self
+      end
+
+      # Gets or sets the active flag.
+      #
+      # The active flag exists because the routing bridge (Routing::Features::Declaratives::Topic)
+      # auto-creates a declaration for every routing topic that calls config(). Some routing
+      # topics - notably Pro pattern-matched virtual topics - must exist in routing but cannot
+      # be managed declaratively (their real Kafka topic names are unknown). Those call
+      # config(active: false) to opt out. Once the routing bridge is retired and declaratives
+      # are defined exclusively via Karafka::App.declaratives.draw, this flag becomes
+      # unnecessary: topics simply won't be declared if they shouldn't be managed.
+      #
+      # @param value [Symbol, Boolean] when :not_set returns current value, otherwise sets it
+      # @return [Boolean] active state
+      def active(value = :not_set)
+        if value == :not_set
+          @active
+        else
+          @active = value
+        end
+      end
+
+      # @return [Boolean] is this topic actively managed via declaratives
+      def active?
+        @active
+      end
+
+      # Gets or sets the partition count
+      # @param value [Symbol, Integer] when :not_set returns current value, otherwise sets it
+      # @return [Integer] partition count
+      def partitions(value = :not_set)
+        if value == :not_set
+          @partitions
+        else
+          @partitions = value
+        end
+      end
+
+      # Gets or sets the replication factor
+      # @param value [Symbol, Integer] when :not_set returns current value, otherwise sets it
+      # @return [Integer] replication factor
+      def replication_factor(value = :not_set)
+        if value == :not_set
+          @replication_factor
+        else
+          @replication_factor = value
+        end
+      end
+
+      # Merges Kafka topic configuration entries into the details hash
+      # @param entries [Hash] topic config entries like 'retention.ms' => 604_800_000
+      def config(entries = {})
+        @details.merge!(entries.transform_keys(&:to_sym))
+      end
+
+      # @return [Hash] hash representation matching the old Config struct's to_h output
+      def to_h
+        {
+          active: @active,
+          partitions: @partitions,
+          replication_factor: @replication_factor,
+          details: @details
+        }
+      end
+    end
+  end
+end

data/lib/karafka/declaratives.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Top-level namespace for declarative topic management (infrastructure-as-code).
+  # Declarative topics define what topics should exist on the Kafka cluster with what
+  # configuration. This is independent of routing, which defines how messages are consumed.
+  module Declaratives
+  end
+end

data/lib/karafka/helpers/interval_runner.rb

@@ -25,14 +25,14 @@ module Karafka
       end
 
       # Runs the requested code if it was not executed previously recently
-      def call
+      def call(...)
         now = monotonic_now
 
         return if now - @last_called_at < @interval
 
         @last_called_at = now
 
-        @block.call
+        @block.call(...)
       end
 
       # Runs the requested code bypassing any time frequencies

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -14,10 +14,33 @@ module Karafka
     class AssignmentsTracker
       include Singleton
 
+      class << self
+        # @return [Hash{Karafka::Routing::Topic => Array<Integer>}]
+        # @see #current
+        def current
+          instance.current
+        end
+
+        # @return [Hash{Karafka::Routing::Topic => Hash{Integer => Integer}}]
+        # @see #generations
+        def generations
+          instance.generations
+        end
+
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Integer]
+        # @see #generation
+        def generation(topic, partition)
+          instance.generation(topic, partition)
+        end
+      end
+
       # Initializes the assignments tracker with empty assignments
       def initialize
         @mutex = Mutex.new
         @assignments = Hash.new { |hash, key| hash[key] = [] }
+        @generations = Hash.new { |h, k| h[k] = {} }
       end
 
       # Returns all the active/current assignments of this given process
@@ -44,10 +67,41 @@ module Karafka
         assignments.freeze
       end
 
-      # Clears all the assignments
+      # Returns the generation counts for all partitions that have ever been assigned
+      #
+      # @return [Hash{Karafka::Routing::Topic => Hash{Integer => Integer}}] topic to partition
+      #   generation mapping. Generation starts at 1 on first assignment and increments on each
+      #   reassignment. Revoked partitions remain in the hash with their last generation value.
+      #
+      # @note Returns a frozen deep copy to prevent external mutation
+      def generations
+        result = {}
+
+        @mutex.synchronize do
+          @generations.each do |topic, partitions|
+            result[topic] = partitions.dup.freeze
+          end
+        end
+
+        result.freeze
+      end
+
+      # Returns the generation count for a specific topic-partition
+      #
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
+      # @return [Integer] generation count (0 if never assigned, 1+ otherwise)
+      def generation(topic, partition)
+        @mutex.synchronize do
+          @generations.dig(topic, partition) || 0
+        end
+      end
+
+      # Clears all the assignments and generations
       def clear
         @mutex.synchronize do
           @assignments.clear
+          @generations.clear
         end
       end
 
@@ -125,7 +179,16 @@ module Karafka
           event[:tpl].to_h.each do |topic, partitions|
             topic = sg.topics.find(topic)
 
-            @assignments[topic] += partitions.map(&:partition)
+            partition_ids = []
+
+            partitions.each do |partition|
+              partition_id = partition.partition
+              partition_ids << partition_id
+              @generations[topic][partition_id] ||= 0
+              @generations[topic][partition_id] += 1
+            end
+
+            @assignments[topic] += partition_ids
             @assignments[topic].sort!
           end
         end

data/lib/karafka/instrumentation/callbacks/consumer_groups/error.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    # Callbacks used to transport things from rdkafka
+    module Callbacks
+      # Consumer-group-specific librdkafka callbacks
+      module ConsumerGroups
+        # Callback that kicks in when consumer error occurs and is published in a background thread
+        class Error
+          include Helpers::ConfigImporter.new(
+            monitor: %i[monitor]
+          )
+
+          # @param subscription_group_id [String]
+          # @param group_id [String] id of the owning group (consumer group today)
+          # @param client_name [String] rdkafka client name
+          def initialize(subscription_group_id, group_id, client_name)
+            @subscription_group_id = subscription_group_id
+            @group_id = group_id
+            @client_name = client_name
+          end
+
+          # Runs the instrumentation monitor with error
+          # @param client_name [String] rdkafka client name
+          # @param error [Rdkafka::Error] error that occurred
+          # @note It will only instrument on errors of the client of our consumer
+          def call(client_name, error)
+            # Emit only errors related to our client Same as with statistics (mor explanation there)
+            return unless @client_name == client_name
+
+            monitor.instrument(
+              "error.occurred",
+              caller: self,
+              subscription_group_id: @subscription_group_id,
+              consumer_group_id: @group_id,
+              group_id: @group_id,
+              type: "librdkafka.error",
+              error: error
+            )
+          rescue => e
+            monitor.instrument(
+              "error.occurred",
+              caller: self,
+              subscription_group_id: @subscription_group_id,
+              consumer_group_id: @group_id,
+              group_id: @group_id,
+              type: "callbacks.error.error",
+              error: e
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/consumer_groups/rebalance.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Consumer-group-specific librdkafka callbacks (rebalance events etc.)
+      module ConsumerGroups
+        # Callback that connects to the librdkafka rebalance callback and converts those events
+        # into our internal events
+        class Rebalance
+          include Helpers::ConfigImporter.new(
+            monitor: %i[monitor]
+          )
+
+          # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
+          #   which we want to manage rebalances
+          # @param client_id [String] id of the client managing this rebalance
+          def initialize(subscription_group, client_id)
+            @subscription_group = subscription_group
+            @client_id = client_id
+          end
+
+          # Publishes an event that partitions are going to be revoked.
+          # At this stage we can still commit offsets, etc.
+          #
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def on_partitions_revoke(tpl)
+            instrument("partitions_revoke", tpl)
+          end
+
+          # Publishes an event that partitions are going to be assigned
+          #
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def on_partitions_assign(tpl)
+            instrument("partitions_assign", tpl)
+          end
+
+          # Publishes an event that partitions were revoked. This is after we've lost them, so no
+          # option to commit offsets.
+          #
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def on_partitions_revoked(tpl)
+            instrument("partitions_revoked", tpl)
+          end
+
+          # Publishes an event that partitions were assigned.
+          #
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def on_partitions_assigned(tpl)
+            instrument("partitions_assigned", tpl)
+          end
+
+          private
+
+          # Publishes info that a rebalance event of a given type has happened
+          #
+          # @param name [String] name of the event
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def instrument(name, tpl)
+            monitor.instrument(
+              "rebalance.#{name}",
+              caller: self,
+              # We keep the id references here for backwards compatibility as some of the monitors
+              # may use the id references
+              subscription_group_id: @subscription_group.id,
+              subscription_group: @subscription_group,
+              # `consumer_group_id:` / `consumer_group:` are kept for backwards compatibility.
+              # `group_id:` / `group:` are the forward-looking polymorphic keys and carry the same
+              # value. The legacy keys will be retired once share groups land.
+              consumer_group_id: @subscription_group.group.id,
+              consumer_group: @subscription_group.group,
+              group_id: @subscription_group.group.id,
+              group: @subscription_group.group,
+              client_id: @client_id,
+              tpl: tpl
+            )
+          rescue => e
+            monitor.instrument(
+              "error.occurred",
+              caller: self,
+              subscription_group_id: @subscription_group.id,
+              consumer_group_id: @subscription_group.group.id,
+              group_id: @subscription_group.group.id,
+              type: "callbacks.rebalance.#{name}.error",
+              client_id: @client_id,
+              error: e
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/callbacks/consumer_groups/statistics.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      module ConsumerGroups
+        # Statistics callback handler
+        # @see `WaterDrop::Instrumentation::Callbacks::Statistics` for details on why we decorate
+        #   those statistics
+        class Statistics
+          include Helpers::ConfigImporter.new(
+            monitor: %i[monitor]
+          )
+
+          # @param subscription_group_id [String]
+          # @param group_id [String] id of the owning group (consumer group today)
+          # @param client_name [String] rdkafka client name
+          def initialize(subscription_group_id, group_id, client_name)
+            @subscription_group_id = subscription_group_id
+            @group_id = group_id
+            @client_name = client_name
+            @statistics_decorator = Karafka::Core::Monitoring::StatisticsDecorator.new
+          end
+
+          # Emits decorated statistics to the monitor
+          # @param statistics [Hash] rdkafka statistics
+          def call(statistics)
+            # Emit only statistics related to our client
+            # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
+            # emit only stats that are related to current producer. Otherwise we would emit all of
+            # all the time.
+            return unless @client_name == statistics["name"]
+
+            monitor.instrument(
+              "statistics.emitted",
+              subscription_group_id: @subscription_group_id,
+              consumer_group_id: @group_id,
+              group_id: @group_id,
+              statistics: @statistics_decorator.call(statistics)
+            )
+          # We need to catch and handle any potential errors coming from the instrumentation pipeline
+          # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
+          # will hang the whole process.
+          rescue => e
+            monitor.instrument(
+              "error.occurred",
+              caller: self,
+              subscription_group_id: @subscription_group_id,
+              consumer_group_id: @group_id,
+              group_id: @group_id,
+              type: "callbacks.statistics.error",
+              error: e
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -28,9 +28,9 @@ module Karafka
       def on_connection_listener_before_fetch_loop(event)
         listener_id = event[:caller].id
         subscription_group = event[:subscription_group]
-        consumer_group_id = subscription_group.consumer_group.id
+        group_id = subscription_group.group.id
         topics = subscription_group.topics.select(&:active?).map(&:name).join(", ")
-        group_details = "#{consumer_group_id}/#{subscription_group.id}"
+        group_details = "#{group_id}/#{subscription_group.id}"
 
         info(
           "[#{listener_id}] Group #{group_details} subscribing to topics: #{topics}"
@@ -93,6 +93,25 @@ module Karafka
         MSG
       end
 
+      # Logs info about the workers pool scaling up
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_worker_scaling_up(event)
+        from = event[:from]
+        to = event[:to]
+        info "Workers pool scaled up from #{from} to #{to} workers"
+      end
+
+      # Logs info about the workers pool scaling down.
+      # The actual size change happens asynchronously as workers pick up nil sentinels and exit.
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_worker_scaling_down(event)
+        from = event[:from]
+        to = event[:to]
+        info "Workers pool scaling down from #{from} to #{to} workers"
+      end
+
       # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
@@ -224,7 +243,7 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details with revoked partitions
       def on_rebalance_partitions_revoked(event)
         revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
-        group_id = event[:consumer_group_id]
+        group_id = event[:group_id]
         client_id = event[:client_id]
         group_prefix = "[#{client_id}] Group #{group_id} rebalance"
 
@@ -242,7 +261,7 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details with assigned partitions
       def on_rebalance_partitions_assigned(event)
         assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
-        group_id = event[:consumer_group_id]
+        group_id = event[:group_id]
         client_id = event[:client_id]
         group_prefix = "[#{client_id}] Group #{group_id} rebalance"
 
@@ -475,8 +494,7 @@ module Karafka
       def error_details(event)
         caller_ref = event[:caller]
 
-        # Collect extra info if it was a consumer related error.
-        # Those come from user code
+        # Collect extra info if it was a consumer related error. Those come from user code
         details = case caller_ref
         when Karafka::BaseConsumer
           extract_consumer_info(caller_ref)
@@ -497,7 +515,7 @@ module Karafka
       # @return [Hash] hash with consumer specific info for details of error
       def extract_consumer_info(consumer)
         {
-          consumer_group: consumer.topic.consumer_group.id,
+          consumer_group: consumer.topic.group.id,
           subscription_group: consumer.topic.subscription_group.id,
           topic: consumer.topic.name,
           partition: consumer.partition,
@@ -510,7 +528,7 @@ module Karafka
       # @return [Hash] hash with client specific info for details of error
       def extract_client_info(client)
         {
-          consumer_group: client.subscription_group.consumer_group.id,
+          consumer_group: client.subscription_group.group.id,
           subscription_group: client.subscription_group.id
         }
       end
@@ -519,7 +537,7 @@ module Karafka
       # @return [Hash] hash with listener specific info for details of error
       def extract_listener_info(listener)
         {
-          consumer_group: listener.subscription_group.consumer_group.id,
+          consumer_group: listener.subscription_group.group.id,
           subscription_group: listener.subscription_group.id
         }
       end

data/lib/karafka/instrumentation/notifications.rb

@@ -108,6 +108,8 @@ module Karafka
         worker.process
         worker.processed
         worker.completed
+        worker.scaling.up
+        worker.scaling.down
 
         error.occurred
       ].freeze