karafka 2.5.3 → 2.5.4.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7b2a6d732bc69e537d14e306a4edc922b6375aa0154dada74527b06efea34fa
-  data.tar.gz: 2e6c1f533f706d70699db822d6d34eac5b6df0c0275bc12366238cc7316a3793
+  metadata.gz: 974c54eac8c3f6045b0eb6f21cde51b48cd8a23af18efd640784870045c01d35
+  data.tar.gz: d22621e0199bf8e1adbfe67f59247d2f8e5ae61a52fc389ea1baa2d4f8401418
 SHA512:
-  metadata.gz: c760fb9e76f6e3af5421cfc0d821d03c3c845c629ea4ab1b2bcfa89c50431320b5cada72482eb48dfc9c27718631e1920ca5133ccd3884daddbc84cc6a7288e7
-  data.tar.gz: ce5466e71edc225b9e11283be5f7e1f877f6222a766dd63f69a235a0739a812f5007da341f5010a71eb2322efebb400fa441a7188f41ac6c244acade0b37c4e4
+  metadata.gz: f81840332313f5914afde17caeaa794085442ac702e62ea4b6a51429cc191ac2cd7562181a55b76bc314fcc0ca6411c27308cc7b0b88637cdece4a8b1e5c75e4
+  data.tar.gz: acbc167ab3314dc884274992cd17d40256c967ab1a2ad141376565b1164b29a11669a9ebabe79125ac7129103d9fb09447441bc992b8540115576fa906b033a5

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka Framework Changelog
 
+## 2.5.4 (Unreleased)
+- [Enhancement] Align Datadog logger listener error handling with the main logger listener by adding missing error types and replacing `UnsupportedCaseError` with generic error logging to support dynamic errors reporting.
+- [Enhancement] Align Appsignal metrics listener `USER_CONSUMER_ERROR_TYPES` with all consumer error types for complete error tracking.
+- [Enhancement] Reduce gem package size by excluding development files (spec, examples, CI configs, dev scripts) from the gem build.
+- [Fix] Multiple route draws don't seem to work.
+- [Fix] Fix internal code compatibility with external libraries that prepend modules to `Messages#each` method (e.g., DataDog tracing) by introducing `Messages#raw` for internal iteration that bypasses patched enumerable methods.
+- [Maintenance] Removed `base64` dependency.
+- [Maintenance] Add `ErrorTypesChecker` spec support module to programmatically verify listener error type coverage against source code definitions.
+
 ## 2.5.3 (2025-11-14)
 - [Enhancement] Dynamically support `librdkafka` fatal errors with correct reported details.
 - [Enhancement] Add `producer` block API to setup for simplified WaterDrop producer configuration without manual producer instance creation, using a transparent ConfigProxy during setup to avoid polluting the permanent config API.
@@ -10,6 +19,7 @@
 - **[EOL]** Remove Rails 7.1 support according to EOL while not blocking Rails 7.1 usage.
 - [Enhancement] Retry on the KIP-848 `stale_member_epoch` error.
 - [Enhancement] Provide `Karafka::Admin.trigger_rebalance` API to programmatically trigger consumer group rebalances for operational purposes.
+- [Enhancement] Provide `Karafka::Admin.plan_topic_replication` API to generate partition reassignment plans for increasing topic replication factors with automatic broker distribution or manual placement, compatible with `kafka-reassign-partitions.sh` tool.
 - [Enhancement] Nest pause configuration under `config.pause.*` namespace (`config.pause.timeout`, `config.pause.max_timeout`, `config.pause.with_exponential_backoff`) while maintaining backwards compatibility with the old flat API (`config.pause_timeout`, etc.) via delegation methods that will be removed in Karafka 2.6.
 - [Enhancement] Detect and track involuntary assignment loss during long-running processing that exceeds `max.poll.interval.ms` via `client.events_poll` event and automatically update `Karafka::App.assignments` to reflect reality.
 - [Enhancement] Extend `Karafka::Admin.read_watermark_offsets` to accept either a single topic with partition or a hash of multiple topics with partitions, using a single consumer instance for improved efficiency when querying multiple partitions.

data/config/locales/errors.yml

@@ -100,6 +100,20 @@ en:
         topics_inclusion: Unknown topic name
         topics_missing: No topics to subscribe to
 
+    admin:
+      replication:
+        missing: needs to be present
+        topic_format: needs to be a non-empty string
+        to_format: needs to be an integer greater than 0
+        brokers_format: needs to be a hash when provided
+        must_be_higher_than_current: target replication factor must be higher than current
+        exceeds_broker_count: target replication factor cannot exceed available broker count
+        missing_partitions: manual assignment missing partitions
+        invalid_partition: partition does not exist in topic
+        wrong_broker_count_for_partition: partition broker count does not match target replication factor
+        duplicate_brokers_for_partition: partition has duplicate brokers
+        invalid_brokers_for_partition: partition references non-existent brokers
+
     routing:
       without_declarative_definition: lacks explicit declarative topics definition
 

data/karafka.gemspec

@@ -21,7 +21,6 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.5.6', '< 2.6.0'
   spec.add_dependency 'karafka-rdkafka', '>= 0.23.1'
   spec.add_dependency 'waterdrop', '>= 2.8.14', '< 3.0.0'
@@ -29,7 +28,19 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 3.2.0'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  gem_files = %w[
+    lib/**/*
+    bin/karafka
+    certs/*
+    config/**/*
+    LICENSE*
+    CHANGELOG.md
+    README.md
+    karafka.gemspec
+  ]
+
+  spec.files = Dir.glob(gem_files) & `git ls-files -z`.split("\x0")
+
   spec.executables   = %w[karafka]
   spec.require_paths = %w[lib]
 

data/lib/karafka/admin/contracts/replication.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Admin
+    # Contracts namespace for Admin validation logic
+    module Contracts
+      # Contract for replication plan parameters
+      class Replication < Karafka::Contracts::Base
+        configure do |config|
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          ).fetch('en').fetch('validations').fetch('admin').fetch('replication')
+        end
+
+        required(:topic) { |val| val.is_a?(String) && !val.empty? }
+        required(:to) { |val| val.is_a?(Integer) && val >= 1 }
+        optional(:brokers) { |val| val.nil? || val.is_a?(Hash) }
+
+        # Validate that target replication factor is higher than current
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:current_rf)
+
+          target_rf = data.fetch(:to)
+          current_rf = data.fetch(:current_rf)
+
+          next if target_rf > current_rf
+
+          [[%w[to], :must_be_higher_than_current]]
+        end
+
+        # Validate that target replication factor doesn't exceed available brokers
+        virtual do |data, errors|
+          next unless errors.empty?
+          next unless data.key?(:broker_count)
+
+          target_rf = data.fetch(:to)
+          broker_count = data.fetch(:broker_count)
+
+          next if target_rf <= broker_count
+
+          [[%w[to], :exceeds_broker_count]]
+        end
+
+        # Validate all partitions are specified in manual broker assignment
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          topic_info = data.fetch(:topic_info)
+          partition_ids = topic_info.fetch(:partitions, []).map { |p| p[:partition_id] }
+
+          missing_partitions = partition_ids - brokers.keys
+          next unless missing_partitions.any?
+
+          [[%w[brokers], :missing_partitions]]
+        end
+
+        # Validate that manual broker assignments reference valid partitions
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          topic_info = data.fetch(:topic_info)
+          partition_ids = topic_info.fetch(:partitions, []).map { |p| p[:partition_id] }
+
+          error_found = nil
+          brokers.each_key do |partition_id|
+            unless partition_ids.include?(partition_id)
+              error_found = [[%w[brokers], :invalid_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate broker count matches target replication factor for each partition
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          target_rf = data.fetch(:to)
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            if broker_ids.size != target_rf
+              error_found = [[%w[brokers], :wrong_broker_count_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate no duplicate brokers assigned to the same partition
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            if broker_ids.size != broker_ids.uniq.size
+              error_found = [[%w[brokers], :duplicate_brokers_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+
+        # Validate all referenced brokers exist in the cluster
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          brokers = data[:brokers]
+          next if brokers.nil?
+          next unless data.key?(:topic_info)
+
+          cluster_info = data.fetch(:cluster_info, {})
+          all_broker_ids = cluster_info.fetch(:brokers, []).map { |b| b[:node_id] }
+
+          error_found = nil
+          brokers.each_value do |broker_ids|
+            invalid_brokers = broker_ids - all_broker_ids
+            if invalid_brokers.any?
+              error_found = [[%w[brokers], :invalid_brokers_for_partition]]
+              break
+            end
+          end
+
+          error_found
+        end
+      end
+    end
+  end
+end