karafka 2.5.2 → 2.5.4.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf868e62a336390b27672a8315accbc535b698cef8e0393a93fbae5e36d3ae87
-  data.tar.gz: 1bafbd50c3e3c0544fc36e34a3ce01808eda30df67b991aba63200a9453cdc35
+  metadata.gz: 974c54eac8c3f6045b0eb6f21cde51b48cd8a23af18efd640784870045c01d35
+  data.tar.gz: d22621e0199bf8e1adbfe67f59247d2f8e5ae61a52fc389ea1baa2d4f8401418
 SHA512:
-  metadata.gz: 57714d5fa6d68c7dde6c911a658793d8898531349922109898038e848cc304ee9316b88023095eb0139b74f8c40a155760e998eb36058e2438d2142c566702fa
-  data.tar.gz: ac31c2972452d8a17e8797d403fb247f85f9179373329a0519e5f9caece5eab6f1dcd3465b2eae73f3f452a9e46f6c30616cb0f6e25712b2a3ae239fcc5de698
+  metadata.gz: f81840332313f5914afde17caeaa794085442ac702e62ea4b6a51429cc191ac2cd7562181a55b76bc314fcc0ca6411c27308cc7b0b88637cdece4a8b1e5c75e4
+  data.tar.gz: acbc167ab3314dc884274992cd17d40256c967ab1a2ad141376565b1164b29a11669a9ebabe79125ac7129103d9fb09447441bc992b8540115576fa906b033a5

data/CHANGELOG.md

@@ -1,9 +1,25 @@
 # Karafka Framework Changelog
 
+## 2.5.4 (Unreleased)
+- [Enhancement] Align Datadog logger listener error handling with the main logger listener by adding missing error types and replacing `UnsupportedCaseError` with generic error logging to support dynamic errors reporting.
+- [Enhancement] Align Appsignal metrics listener `USER_CONSUMER_ERROR_TYPES` with all consumer error types for complete error tracking.
+- [Enhancement] Reduce gem package size by excluding development files (spec, examples, CI configs, dev scripts) from the gem build.
+- [Fix] Multiple route draws don't seem to work.
+- [Fix] Fix internal code compatibility with external libraries that prepend modules to `Messages#each` method (e.g., DataDog tracing) by introducing `Messages#raw` for internal iteration that bypasses patched enumerable methods.
+- [Maintenance] Removed `base64` dependency.
+- [Maintenance] Add `ErrorTypesChecker` spec support module to programmatically verify listener error type coverage against source code definitions.
+
+## 2.5.3 (2025-11-14)
+- [Enhancement] Dynamically support `librdkafka` fatal errors with correct reported details.
+- [Enhancement] Add `producer` block API to setup for simplified WaterDrop producer configuration without manual producer instance creation, using a transparent ConfigProxy during setup to avoid polluting the permanent config API.
+- [Change] Require `waterdrop` `>=` `2.8.14` to support new features.
+- [Change] Require `karafka-rdkafka` `>=` `0.23.1` to support new rebalance protocol.
+
 ## 2.5.2 (2025-10-31)
 - **[EOL]** Remove Rails 7.1 support according to EOL while not blocking Rails 7.1 usage.
 - [Enhancement] Retry on the KIP-848 `stale_member_epoch` error.
 - [Enhancement] Provide `Karafka::Admin.trigger_rebalance` API to programmatically trigger consumer group rebalances for operational purposes.
+- [Enhancement] Provide `Karafka::Admin.plan_topic_replication` API to generate partition reassignment plans for increasing topic replication factors with automatic broker distribution or manual placement, compatible with `kafka-reassign-partitions.sh` tool.
 - [Enhancement] Nest pause configuration under `config.pause.*` namespace (`config.pause.timeout`, `config.pause.max_timeout`, `config.pause.with_exponential_backoff`) while maintaining backwards compatibility with the old flat API (`config.pause_timeout`, etc.) via delegation methods that will be removed in Karafka 2.6.
 - [Enhancement] Detect and track involuntary assignment loss during long-running processing that exceeds `max.poll.interval.ms` via `client.events_poll` event and automatically update `Karafka::App.assignments` to reflect reality.
 - [Enhancement] Extend `Karafka::Admin.read_watermark_offsets` to accept either a single topic with partition or a hash of multiple topics with partitions, using a single consumer instance for improved efficiency when querying multiple partitions.

data/config/locales/errors.yml

@@ -100,6 +100,20 @@ en:
         topics_inclusion: Unknown topic name
         topics_missing: No topics to subscribe to
 
+    admin:
+      replication:
+        missing: needs to be present
+        topic_format: needs to be a non-empty string
+        to_format: needs to be an integer greater than 0
+        brokers_format: needs to be a hash when provided
+        must_be_higher_than_current: target replication factor must be higher than current
+        exceeds_broker_count: target replication factor cannot exceed available broker count
+        missing_partitions: manual assignment missing partitions
+        invalid_partition: partition does not exist in topic
+        wrong_broker_count_for_partition: partition broker count does not match target replication factor
+        duplicate_brokers_for_partition: partition has duplicate brokers
+        invalid_brokers_for_partition: partition references non-existent brokers
+
     routing:
       without_declarative_definition: lacks explicit declarative topics definition
 

data/karafka.gemspec

@@ -21,15 +21,26 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.5.6', '< 2.6.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.22.0'
-  spec.add_dependency 'waterdrop', '>= 2.8.9', '< 3.0.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.23.1'
+  spec.add_dependency 'waterdrop', '>= 2.8.14', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.2.0'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  gem_files = %w[
+    lib/**/*
+    bin/karafka
+    certs/*
+    config/**/*
+    LICENSE*
+    CHANGELOG.md
+    README.md
+    karafka.gemspec
+  ]
+
+  spec.files = Dir.glob(gem_files) & `git ls-files -z`.split("\x0")
+
   spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
 

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -21,7 +21,7 @@ module ActiveJob
     # - No Rails: Inherit from Object (standalone ActiveJob usage)
     #
     # @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
-    base = if defined?(Rails) && defined?(Rails::VERSION)
+    base = if defined?(Rails::VERSION)
              (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter)
            else
              # Fallback when Rails is not loaded
@@ -31,7 +31,7 @@ module ActiveJob
     # Karafka adapter for enqueuing jobs
     # This is here for ease of integration with ActiveJob.
     class KarafkaAdapter < base
-      include Karafka::Helpers::ConfigImporter.new(
+      include ::Karafka::Helpers::ConfigImporter.new(
         dispatcher: %i[internal active_job dispatcher]
       )
 

data/lib/karafka/active_job/consumer.rb

@@ -5,7 +5,7 @@ module Karafka
   module ActiveJob
     # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
     # It marks the offset after each message, so we make sure, none of the jobs is executed twice
-    class Consumer < ::Karafka::BaseConsumer
+    class Consumer < Karafka::BaseConsumer
       include Helpers::ConfigImporter.new(
         deserializer: %i[internal active_job deserializer]
       )
@@ -14,7 +14,7 @@ module Karafka
       # @note ActiveJob does not support batches, so we just run one message after another
       def consume
         messages.each do |message|
-          break if Karafka::App.stopping?
+          break if ::Karafka::App.stopping?
 
           consume_job(message)
 

data/lib/karafka/active_job/current_attributes.rb

@@ -23,8 +23,8 @@ module Karafka
           .each { |expandable| expandable.class_attribute :_cattr_klasses, default: {} }
 
         # Do not double inject in case of running persist multiple times
-        Dispatcher.prepend(Persistence) unless Dispatcher.ancestors.include?(Persistence)
-        Consumer.prepend(Loading) unless Consumer.ancestors.include?(Loading)
+        Dispatcher.prepend(Persistence) unless Dispatcher <= Persistence
+        Consumer.prepend(Loading) unless Consumer <= Loading
 
         klasses.map(&:to_s).each do |stringified_klass|
           # Prevent registering same klass multiple times

data/lib/karafka/active_job/deserializer.rb

@@ -43,7 +43,7 @@ module Karafka
       #
       # @param job [ActiveJob::Base, #serialize] job to serialize. The job must respond to
       #   #serialize which returns a Hash of job attributes. When CurrentAttributes are used,
-      #   this may be a JobWrapper instance instead of the original ActiveJob::Base.
+      #   this may be a JobWrapper instance instead of the original ::ActiveJob::Base.
       # @return [String] serialized job payload
       def serialize(job)
         ::ActiveSupport::JSON.encode(job.serialize)

data/lib/karafka/active_job/dispatcher.rb

@@ -19,7 +19,7 @@ module Karafka
 
       # @param job [ActiveJob::Base] job
       def dispatch(job)
-        ::Karafka.producer.public_send(
+        Karafka.producer.public_send(
           fetch_option(job, :dispatch_method, DEFAULTS),
           topic: job.queue_name,
           payload: serialize_job(job)
@@ -43,7 +43,7 @@ module Karafka
         end
 
         dispatches.each do |type, messages|
-          ::Karafka.producer.public_send(
+          Karafka.producer.public_send(
             type,
             messages
           )

data/lib/karafka/admin/configs/resource.rb

@@ -40,10 +40,16 @@ module Karafka
         OPERATIONS_TYPES_MAP.each do |op_name, op_value|
           # Adds an outgoing operation to a given resource of a given type
           # Useful since we alter in batches and not one at a time
+          #
+          # For example, when op_name is :set and op_value is 0:
+          #   def set(name, value)
+          #     @operations[0] << Config.new(name: name, value: value.to_s)
+          #   end
+          default_value = op_name == :delete ? ' = nil' : ''
           class_eval <<~RUBY, __FILE__, __LINE__ + 1
             # @param name [String] name of the config to alter
             # @param value [String] value of the config
-            def #{op_name}(name, value #{op_name == :delete ? ' = nil' : ''})
+            def #{op_name}(name, value#{default_value})
               @operations[#{op_value}] << Config.new(name: name, value: value.to_s)
             end
           RUBY

data/lib/karafka/admin/consumer_groups.rb

@@ -80,15 +80,13 @@ module Karafka
               case casted_position
               # Earliest is not always 0. When compacting/deleting it can be much later, that's why
               # we fetch the oldest possible offset
-              when 'earliest'
+              # false is treated the same as 'earliest'
+              when 'earliest', false
                 LONG_TIME_AGO
               # Latest will always be the high-watermark offset and we can get it just by getting
               # a future position
               when 'latest'
                 Time.now + DAY_IN_SECONDS
-              # Same as `'earliest'`
-              when false
-                LONG_TIME_AGO
               # Regular offset case
               else
                 position
@@ -307,14 +305,14 @@ module Karafka
         # Reads lags and offsets for given topics in the context of consumer groups defined in the
         #   routing
         #
-        # @param consumer_groups_with_topics [Hash<String, Array<String>>] hash with consumer
+        # @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
         #   groups names with array of topics to query per consumer group inside
         # @param active_topics_only [Boolean] if set to false, when we use routing topics, will
         #   select also topics that are marked as inactive in routing
         #
-        # @return [Hash<String, Hash<Integer, <Hash<Integer>>>>] hash where the top level keys are
-        #   the consumer groups and values are hashes with topics and inside partitions with lags
-        #   and offsets
+        # @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
+        #   level keys are the consumer groups and values are hashes with topics and inside
+        #   partitions with lags and offsets
         #
         # @note For topics that do not exist, topic details will be set to an empty hash
         #

data/lib/karafka/admin/contracts/replication.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Admin
+    # Contracts namespace for Admin validation logic
+    module Contracts
+      # Contract for replication plan parameters
+      class Replication < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          ).fetch('en').fetch('validations').fetch('admin').fetch('replication')
+        end
+
+        required(:topic) { |val| val.is_a?(String) && !val.empty? }
+        required(:to) { |val| val.is_a?(Integer) && val >= 1 }
+        optional(:brokers) { |val| val.nil? || val.is_a?(Hash) }
+
+        # Validate that target replication factor is higher than current
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:current_rf)
+
+          target_rf = data.fetch(:to)
+          current_rf = data.fetch(:current_rf)
+
+          next if target_rf > current_rf
+
+          [[%w[to], :must_be_higher_than_current]]
+        end
+
+        # Validate that target replication factor doesn't exceed available brokers
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:broker_count)
+
+          target_rf = data.fetch(:to)
+          broker_count = data.fetch(:broker_count)
+
+          next if target_rf <= broker_count
+
+          [[%w[to], :exceeds_broker_count]]
+        end
+
+        # Validate all partitions are specified in manual broker assignment
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          topic_info = data.fetch(:topic_info)
+          partition_ids = topic_info.fetch(:partitions, []).map { |p| p[:partition_id] }
+
+          missing_partitions = partition_ids - brokers.keys
+          next unless missing_partitions.any?
+
+          [[%w[brokers], :missing_partitions]]
+        end
+
+        # Validate that manual broker assignments reference valid partitions
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          topic_info = data.fetch(:topic_info)
+          partition_ids = topic_info.fetch(:partitions, []).map { |p| p[:partition_id] }
+
+          error_found = nil
+          brokers.each_key do |partition_id|
+            unless partition_ids.include?(partition_id)
+              error_found = [[%w[brokers], :invalid_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate broker count matches target replication factor for each partition
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          target_rf = data.fetch(:to)
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            if broker_ids.size != target_rf
+              error_found = [[%w[brokers], :wrong_broker_count_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate no duplicate brokers assigned to the same partition
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            if broker_ids.size != broker_ids.uniq.size
+              error_found = [[%w[brokers], :duplicate_brokers_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate all referenced brokers exist in the cluster
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          cluster_info = data.fetch(:cluster_info, {})
+          all_broker_ids = cluster_info.fetch(:brokers, []).map { |b| b[:node_id] }
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            invalid_brokers = broker_ids - all_broker_ids
+            if invalid_brokers.any?
+              error_found = [[%w[brokers], :invalid_brokers_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+      end
+    end
+  end
+end