karafka 2.5.9 → 2.5.10.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a43645cb0713b1565aa8ff9b3779d5d19225f08d2449e816ab17aaa532a6ffd
-  data.tar.gz: 438e1dcb6d6b65fd29e2d4f2feb6cee80cf8c629cba273b3e2936637379cda75
+  metadata.gz: 953c11ca73eda62bac76aaca94946b50b880a54e50756b0665da68155c9ad68d
+  data.tar.gz: a5d200ac66bc102d6dac680aaa03df1cc897113dc2af52f4684f3f7128608340
 SHA512:
-  metadata.gz: 44a2d60d274f70321846a4190e007f4d5c75f22e995835fe92cf29a5a658ed7eb99bd96147d6cfa08e9821676ca7064fcd095ecf17b6ac4014c018d885621e27
-  data.tar.gz: 4eb201aeb1e1d416008913c58b7f39637dce2342506f0032d8556b285025cf23f1475f8cf08f2d91a4ee00a9241902e26fc8ca7297547a41cd0b1126d09029ac
+  metadata.gz: 96d8087bda95065a1b56f5a525b9dd53c9ec71d01d9dd51f56403f7a14fd5ee406f2c40a366e1bb607495ebea8ce76ddd379df08e8e76594cdfae5905ff9c3d0
+  data.tar.gz: 393f0264db679e7b71eeb2fe6c419090d6ff46a43354789fcef2e1e7e6baec3c839583982129669d02a6eae1d245396f48c0dcfebe475946ca2aa97dadce90ed

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Karafka Framework Changelog
 
+## 2.5.10 (Unreleased)
+- **[Feature]** Add `Processing::WorkersPool` with dynamic thread pool scaling via `#scale`, `nil` sentinel-based worker exit for downscaling, and `worker.scaling.up`/`worker.scaling.down` instrumentation events.
+- [Enhancement] Add per-partition generation tracking to `AssignmentsTracker` to distinguish first-time assignments from reassignments.
+- [Enhancement] Make liveness listeners fiber-safe.
+- [Fix] Include orphaned node detection in Pro swarm liveness listener.
+- [Fix] Report liveness on `connection.listener.before_fetch_loop` so nodes send an initial healthy report before the first consumption.
+- [Fix] Fire liveness events during `wait_pinging` so nodes continue reporting health status during shutdown with active LRJ jobs.
+- [Maintenance] Use namespaced topic naming format in all integration specs for consistent traceability.
+- [Fix] Fix `DataCollector::SPEC_HASH` producing non-deterministic hashes for pristine and poro specs by passing the original spec path via `KARAFKA_SPEC_PATH` env var.
+- [Maintenance] Add `bin/tests_topics_hashes` script for looking up spec files by their topic name hash prefix.
+- [Change] Require `karafka-rdkafka` `>=` `0.26.1` to support upcoming features relying on low-level Rdkafka APIs.
+
 ## 2.5.9 (2026-03-30)
 - [Enhancement] Validate that `statistics.interval.ms` is not zero when dynamic multiplexing is enabled (Pro).
 - [Fix] Fix swarm liveness reporting to also use `on_connection_listener_fetch_loop` so nodes stay alive when `statistics.interval.ms` is disabled.

data/certs/expired.txt

@@ -1,2 +1,85 @@
 # Expired/revoked license checksums (SHA256 of license token)
 # One checksum per line, lines starting with # are comments
+
+01603767339fd25cf874e6fe29315fb27b6f6bbdf2e769c3b88e6efcf4096d37
+024e2c943693c4ea9eea852eb02d32acc9cb34df21aea90af5ab39dcf3b1d8cf
+0371303ba704d8f256b40932aa7840a1adb1cdc6fd032ac902794320d0d35ff1
+051dc2d8176df8d3bde1595e01af786a3f58ccab4cc48743d624e68125d80bd6
+06e3d91c9881ebf9686f3492e53400adc2c6a5fd4a409f6775cd36094b24f181
+0ae8daf74b8ccd6c4828ed707edf0d05df3e7d328e91ddc973e160ae8b120787
+0b4420e202d9c6075bb108ac4bd5917e35e89aa7d1741afaab93ca07839d06fc
+11936f8e0637565a309157d1fd4e84a3c44e5a3f9f226435e7d38bc43e452db6
+122ecfd0493596239995b9e7425046737b4893e28e33f31815eb4727c3f607d9
+17c14c38e10d0d2bd4a9c847a2c23fd0726459337af1d9815b6e34d1fd9af8be
+227f48f0b22974c43f120de28d5ca33a77e58d33064f9a73748501ba700f1474
+24d59b10d982e5642dfda3ec54cd56dda9022927c89584bbe12d31f3df28e17d
+2634761cc75479548003f941346f0a51a6c535415696f386b40ae28f88f63efa
+284967207a93e9c989d05ad6e5bb0dbe3e0a8a871c3fba48324558f127166bda
+2e2b4ccd902e43feb418cc8b3aed5874ec194ae8495ded864aff917538d91f7a
+39aeae710122bf14729a05f24f5bd702d2a1127e5c7fb9bd90de0e20890575dc
+3ada575caaf0b8bdeca0686a93074de9ed830751e6f46251849475f70e84e408
+4367e57ca36a4b447f23745d76611514df61b56611225be4c8f9a90560bbace9
+47380207ca3041b14c7d4c81a69f820852069ecab4d80a068d39133f61a6e641
+4d49b497ece81604358afc8e9adc7d1163995c4c08446bcd5c0bbfe3d7ebc1e7
+4d97c95c46e51105705964aecbfa5219120295e7d0a172f5c098d12c3a5cbf7e
+60171be4b327d48b2445b00f1fca3d2a3554d06c69f63f3e97742b29030a0a69
+630907de3c6bf258bfc2c51d768d51ca446ebd49fb29b29a0b24d363d46d1a6e
+68db4a9700f6cc3494f8eac1e0df2471f11ef7312e7b58dae839b193e226fae5
+6a06fb85cee01a27a5e819e464b0b5fd84051d234540d11774bc17e90c91abd4
+6d79bcac25c0026dad0f1871ccae79fc07d696df2e0f080ce1d1038454545a05
+6e2ed71a0925d59815c49ea1e72dfd0ba28bdbb57f93d81fa34a31c50db53830
+711a3cced55062707b64f6d652e107d0f1318fd9f3762c732fad23d04884cdd2
+71c2d2ab89a11010c41c9938fbc2565297364b4e8378e78ddf58b22ea2c43ca2
+72b19e6e274b1f36799eea65b07f4849db44c82315647266608ebdb6fde0cad6
+731c0d75d9b47cca49d9d95fc8d291fda452383f920e70849373959892443329
+77b6974b96701d82a0670c79830543e345bf0a00d9e5b5a1bbf8443439d75987
+77e461a9a4534870cfb7cca350d56ed0a504643bb39eeb750b338179d05069a9
+7c06718c4314e28561f337b3ecc66fb012c2510f1ed5fcb82dbab5ba009a8f7d
+7ca5a6183ed9c97e4da2c7030f809bfb32da49319d106377bd0e6d7d52734141
+7ce4cade640a4f34e2909360c4c0ceb8a647b7bfead7a57880d8ce2bf6eea26c
+7d3a2cb0ab08f2f25324124973370da30210dab04abaeeed97192576238a03e0
+7dc790fd8bf18e663c2e4276f4362b0c14795945b508807ef03a92ece266cff8
+81e69732cc6e9c5266fffea08fd1c69fd12c1d841b979723a77533c5efef244e
+839f4a449e0d68acccc0f6a2d0209d2d137375d740ae6d0244dde17e63131fbb
+85d26e4fedf364e20ea33c1b0102d34cea19156243fe781ddf597f6677cf2de3
+8609d26341a5c4e908af8b63bcc6895d99d8cc13f0403dc7b36a2920bb8e048a
+8d3e09a94bd00710a49b186be832f74cc3340ec3d87e780e10159e5e8d048b81
+8e933669352c272935c310d7ce798b3ea2687611378b978e412491e7fddfce4d
+8f3190524285a5c0a0d1e24bdc2a5c417416c2cb92b9da26f0e492aaa4600bd7
+9221a49b42e05d39e1939cf9353805abea2042457c2247859e3274976ac2fbc6
+96c8dde1a7f938899f3f5bd764eff295f090e1c40635e7efd25d69ef55881db2
+9b0b791259e86740d4a2cab736802614938b7043479ac8770c570b1b37f38fb7
+a0db6c5e7c8714a2d33adff3b69d2d688d5a34e6d5ba871d1cbdfcba3edf1e89
+a7ce882d45a00ad82e653b60d14f1a1ebefdc9033d6cc60ed0a2ed507949a9a2
+ac2941c112f397543fde83cc1b0dcf70ec34d77e149d3e90ad07a0f1bc809933
+adea773fadaaaeadf0f20643e84ae6c40bcb55fb73ecee5894a41836453e4abb
+b1411e92ad939b59c898e78520c8ec82547950493a45004b747f12820aa7d390
+b3984622fbc6ba91a2180283a6ada7e1bbe01fb705dbaae37c831e9c893b8e00
+b5ed1ff168e86300735e59dc2786be6d535fc9e0f3c6c8a76bc4354cbbed4ae0
+b9e38e9c1b34db847a4b82ae35382b06e61a5872aa8c24f988d7d5246ded4648
+bdc4f155153a59c25c7c52932f77973487d6336e99eb0beeb90edf4a4120a94a
+bf6c5c58b02fa5e7c7dc845213ad943c5b6613f12ac6012be00b1484bb85aa04
+c011c3d168a159c135ef35bf55121edaf1a4cc94f663a1f461595776b0313160
+c311cb6cfb38a8b2e83b3080c1f87444a52f3f645be8337559a123aaacf1344a
+c87222a94f98c3a2191dda9254bd6ed031fd8ca190f75b40c67a0fcd4b56f795
+ca53af70cd7e67abbb9dbf0a8da6788a9b2192aa3a5a1dd8720aaa68a2511722
+d02da0cbd2c1d83432a2cee7984dd311249692373554002d1d543dd5007f6a58
+d13295e597120b88bf4c4f36de63d0aa3e5a68e3578c09a2e026bd1f6aaaf04f
+d862dd776d5b42a73a63574dc00187c587403101cd211ed348fc422f6460844e
+d9f25b56eae6a62b9383c068c2a7c22ef6b051f1cf4137164ba3dd1baf0bac61
+db1f2c50f3dc6956c498b06f67f82412a76f0b138265a55713076b2a4a35725f
+ded7405c91e9bb1994c58c728f83b705fdc5e816ce0cb3fdf16887127fff27f0
+dfe08da10ae5e74e9bbf648261860f3b94201a11ab4269cd4db7c73017452dde
+dff7372b5ee3136af892561507602423bbae42d12d3a65277ae2af1add666723
+e1a040c0c9a6c5e4603d9aebbc4201075238b47446348ab635b53704b3095a63
+e3b94496b65bb626e723637926e729a33be673fe0355d964306bde563ff6fbf7
+e577d74987f1d31c10fc4ec67dd74084486514f844668d7d01bd4cad2d8f09c4
+e7a91a5ce0e68b888026bbddb57c92c7d5ad61e9416c8188721eee0a8ba85bc1
+eda2ab2c99f769d646dd6fff564af9ec6f32e4462390e3b3246bf953cd7a9c70
+f031aa98b2375c6573f6f775f6000dc81e07b1034694742a900a9115eb124900
+f1ade1306fd64c2e32b15e505d87c0a9ec7a8392f5bea014fc4e70acd17f9935
+f397ccd272b6ee7b33595aa93a6e40b725af265b681efca05bb26f9a7766d6c2
+f54723b14f7cfc4e3c88b946cab881b907dd5ac8372b4600213241ab82778979
+f9cc7b7d9c7d81ba477a70749d166edbc2b44f1714096fce10118bb18fd9696c
+fb417a807d7595a4aff44cc79712e9bc887181e64eb9a98fe9f200b585856ae7
+fd5009f463fe0c38cfe66677ff9b8e5031cbc4ce354883c99c392109ae8ae3b8

data/config/locales/errors.yml

@@ -25,6 +25,9 @@ en:
           Decrease max_wait_time or increase node_report_timeout
         kafka_format: needs to be a filled hash
         key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+        managed_key_not_supported: >
+          this kafka setting is managed internally by Karafka and should not be set directly.
+          Setting it manually may cause misbehaviours and other unexpected issues
         pause.timeout_max_timeout_vs_pause_max_timeout: pause.timeout must be less or equal to pause.max_timeout
         shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
         worker_thread_priority_format: must be between -3 and 3

data/karafka.gemspec

@@ -21,8 +21,8 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency "karafka-core", ">= 2.5.6", "< 2.6.0"
-  spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
+  spec.add_dependency "karafka-core", ">= 2.5.13", "< 2.6.0"
+  spec.add_dependency "karafka-rdkafka", ">= 0.26.1"
   spec.add_dependency "waterdrop", ">= 2.8.14", "< 3.0.0"
   spec.add_dependency "zeitwerk", "~> 2.3"
 

data/lib/karafka/connection/client.rb

@@ -401,10 +401,14 @@ module Karafka
       #   - OAUTHBEARER token refresh callbacks
       #
       # @param timeout [Integer] number of milliseconds to wait on events or 0 not to wait.
+      # @param safe [Boolean] when true, rescues Rdkafka::RdkafkaError so callers in
+      #   shutdown/quiet paths do not trigger a full listener reset. When shutting down, errors
+      #   at this layer are not relevant enough. We want to log them but we should not propagate
+      #   them any further.
       #
       # @note It is non-blocking when timeout 0 and will not wait if queue empty. It costs up to
       #   2ms when no callbacks are triggered.
-      def events_poll(timeout = 0)
+      def events_poll(timeout = 0, safe: false)
         kafka.events_poll(timeout)
 
         # Emit event for monitoring - happens once per tick_interval (default 5s)
@@ -414,6 +418,8 @@ module Karafka
           caller: self,
           subscription_group: @subscription_group
         )
+      rescue Rdkafka::RdkafkaError
+        safe ? nil : raise
       end
 
       # Returns pointer to the consumer group metadata. It is used only in the context of

data/lib/karafka/connection/listener.rb

@@ -50,7 +50,7 @@ module Karafka
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         @partitioner = partitioner_class.new(subscription_group)
         @scheduler = scheduler
-        @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
+        @events_poller = Helpers::IntervalRunner.new { |**opts| @client.events_poll(**opts) }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
@@ -499,6 +499,7 @@ module Karafka
         until wait_until.call
           @client.ping
           @scheduler.on_manage
+          @events_poller.call(safe: true)
 
           after_ping.call
           sleep(0.2)

data/lib/karafka/helpers/interval_runner.rb

@@ -25,14 +25,14 @@ module Karafka
       end
 
       # Runs the requested code if it was not executed previously recently
-      def call
+      def call(...)
         now = monotonic_now
 
         return if now - @last_called_at < @interval
 
         @last_called_at = now
 
-        @block.call
+        @block.call(...)
       end
 
       # Runs the requested code bypassing any time frequencies

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -14,10 +14,33 @@ module Karafka
     class AssignmentsTracker
       include Singleton
 
+      class << self
+        # @return [Hash{Karafka::Routing::Topic => Array<Integer>}]
+        # @see #current
+        def current
+          instance.current
+        end
+
+        # @return [Hash{Karafka::Routing::Topic => Hash{Integer => Integer}}]
+        # @see #generations
+        def generations
+          instance.generations
+        end
+
+        # @param topic [Karafka::Routing::Topic]
+        # @param partition [Integer]
+        # @return [Integer]
+        # @see #generation
+        def generation(topic, partition)
+          instance.generation(topic, partition)
+        end
+      end
+
       # Initializes the assignments tracker with empty assignments
       def initialize
         @mutex = Mutex.new
         @assignments = Hash.new { |hash, key| hash[key] = [] }
+        @generations = Hash.new { |h, k| h[k] = {} }
       end
 
       # Returns all the active/current assignments of this given process
@@ -44,10 +67,41 @@ module Karafka
         assignments.freeze
       end
 
-      # Clears all the assignments
+      # Returns the generation counts for all partitions that have ever been assigned
+      #
+      # @return [Hash{Karafka::Routing::Topic => Hash{Integer => Integer}}] topic to partition
+      #   generation mapping. Generation starts at 1 on first assignment and increments on each
+      #   reassignment. Revoked partitions remain in the hash with their last generation value.
+      #
+      # @note Returns a frozen deep copy to prevent external mutation
+      def generations
+        result = {}
+
+        @mutex.synchronize do
+          @generations.each do |topic, partitions|
+            result[topic] = partitions.dup.freeze
+          end
+        end
+
+        result.freeze
+      end
+
+      # Returns the generation count for a specific topic-partition
+      #
+      # @param topic [Karafka::Routing::Topic]
+      # @param partition [Integer]
+      # @return [Integer] generation count (0 if never assigned, 1+ otherwise)
+      def generation(topic, partition)
+        @mutex.synchronize do
+          @generations.dig(topic, partition) || 0
+        end
+      end
+
+      # Clears all the assignments and generations
       def clear
         @mutex.synchronize do
           @assignments.clear
+          @generations.clear
         end
       end
 
@@ -125,7 +179,16 @@ module Karafka
           event[:tpl].to_h.each do |topic, partitions|
             topic = sg.topics.find(topic)
 
-            @assignments[topic] += partitions.map(&:partition)
+            partition_ids = []
+
+            partitions.each do |partition|
+              partition_id = partition.partition
+              partition_ids << partition_id
+              @generations[topic][partition_id] ||= 0
+              @generations[topic][partition_id] += 1
+            end
+
+            @assignments[topic] += partition_ids
             @assignments[topic].sort!
           end
         end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -93,6 +93,25 @@ module Karafka
         MSG
       end
 
+      # Logs info about the workers pool scaling up
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_worker_scaling_up(event)
+        from = event[:from]
+        to = event[:to]
+        info "Workers pool scaled up from #{from} to #{to} workers"
+      end
+
+      # Logs info about the workers pool scaling down.
+      # The actual size change happens asynchronously as workers pick up nil sentinels and exit.
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_worker_scaling_down(event)
+        from = event[:from]
+        to = event[:to]
+        info "Workers pool scaling down from #{from} to #{to} workers"
+      end
+
       # Prints info about a consumer pause occurrence. Irrelevant if user or system initiated.
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload

data/lib/karafka/instrumentation/notifications.rb

@@ -108,6 +108,8 @@ module Karafka
         worker.process
         worker.processed
         worker.completed
+        worker.scaling.up
+        worker.scaling.down
 
         error.occurred
       ].freeze

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -328,10 +328,8 @@ module Karafka
 
           # Sends minute based probing metrics
           def minute_probe
-            concurrency = Karafka::App.config.concurrency
-
             count("processes_count", 1, {})
-            count("threads_count", concurrency, {})
+            count("threads_count", Karafka::Server.workers.size, {})
           end
         end
       end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -164,7 +164,7 @@ module Karafka
             jq_stats = event[:jobs_queue].statistics
 
             tags = default_tags
-            gauge("worker.total_threads", Karafka::App.config.concurrency, tags: tags)
+            gauge("worker.total_threads", Karafka::Server.workers.size, tags: tags)
             histogram("worker.processing", jq_stats[:busy], tags: tags)
             histogram("worker.enqueued_jobs", jq_stats[:enqueued], tags: tags)
           end

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -158,36 +158,39 @@ module Karafka
             @mutex.synchronize(&)
           end
 
-          # @return [Integer] object id of the current thread
-          def thread_id
-            Thread.current.object_id
+          # @return [Integer] object id of the current fiber
+          # @note We use fiber object id instead of thread object id to ensure fiber-safety.
+          #   Multiple fibers can run on the same thread, and using thread id would cause them
+          #   to overwrite each other's timestamps.
+          def fiber_id
+            Fiber.current.object_id
           end
 
-          # Update the polling tick time for current thread
+          # Update the polling tick time for current fiber
           def mark_polling_tick
             synchronize do
-              @pollings[thread_id] = monotonic_now
+              @pollings[fiber_id] = monotonic_now
             end
           end
 
-          # Clear current thread polling time tracker
+          # Clear current fiber polling time tracker
           def clear_polling_tick
             synchronize do
-              @pollings.delete(thread_id)
+              @pollings.delete(fiber_id)
             end
           end
 
           # Update the processing tick time
           def mark_consumption_tick
             synchronize do
-              @consumptions[thread_id] = monotonic_now
+              @consumptions[fiber_id] = monotonic_now
             end
           end
 
-          # Clear current thread consumption time tracker
+          # Clear current fiber consumption time tracker
           def clear_consumption_tick
             synchronize do
-              @consumptions.delete(thread_id)
+              @consumptions.delete(fiber_id)
             end
           end
 

data/lib/karafka/pro/loader.rb

@@ -113,6 +113,8 @@ module Karafka
         def reconfigure(config)
           icfg = config.internal
 
+          Karafka::Setup::DefaultsInjector.singleton_class.prepend(Setup::DefaultsInjector)
+
           icfg.cli.contract = Cli::Contracts::Server.new
 
           # Use manager that supports multiplexing

data/lib/karafka/pro/setup/defaults_injector.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    # Namespace for Pro setup components
+    module Setup
+      # Pro defaults injector that extends the OSS defaults with Pro-specific settings
+      module DefaultsInjector
+        # Pro-specific consumer kafka defaults
+        # These defaults are carefully tuned to work with Pro's internal statistics aggregation,
+        # the Web UI dashboard, and the performance tracker. They depend on Pro's extended
+        # instrumentation pipeline and should not be applied outside of Pro as they may cause
+        # incomplete or inconsistent metrics collection and other unexpected behaviours.
+        CONSUMER_KAFKA_DEFAULTS = {
+          "statistics.unassigned.include": false
+        }.freeze
+
+        private_constant :CONSUMER_KAFKA_DEFAULTS
+
+        # Pro actively manages these keys via its own DefaultsInjector so users are allowed
+        # to set them if needed.
+        #
+        # @return [Set<Symbol>] empty set since Pro handles these keys
+        def managed_keys
+          @managed_keys ||= Set.new
+        end
+
+        # Enriches consumer kafka config with Pro-specific defaults
+        # @param kafka_config [Hash] kafka scoped config
+        def consumer(kafka_config)
+          super
+
+          CONSUMER_KAFKA_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -73,6 +73,13 @@ module Karafka
           super()
         end
 
+        # Report status before the first fetch loop iteration so the supervisor gets an initial
+        # healthy report even if the first consumption takes longer than the report timeout.
+        # @param _event [Karafka::Core::Monitoring::Event]
+        def on_connection_listener_before_fetch_loop(_event)
+          report_status
+        end
+
         # Tick on each fetch and report liveness so it works even when statistics are disabled
         #
         # @param _event [Karafka::Core::Monitoring::Event]
@@ -143,42 +150,47 @@ module Karafka
           periodically do
             return unless node
 
+            Kernel.exit!(orphaned_exit_code) if node.orphaned?
+
             current_status = status
 
             current_status.positive? ? node.unhealthy(current_status) : node.healthy
           end
         end
 
-        # @return [Integer] object id of the current thread
-        def thread_id
-          Thread.current.object_id
+        # @return [Integer] object id of the current fiber
+        # @note We use fiber object id instead of thread object id to ensure fiber-safety.
+        #   Multiple fibers can run on the same thread, and using thread id would cause them
+        #   to overwrite each other's timestamps.
+        def fiber_id
+          Fiber.current.object_id
         end
 
-        # Update the polling tick time for current thread
+        # Update the polling tick time for current fiber
         def mark_polling_tick
           synchronize do
-            @pollings[thread_id] = monotonic_now
+            @pollings[fiber_id] = monotonic_now
           end
         end
 
-        # Clear current thread polling time tracker
+        # Clear current fiber polling time tracker
         def clear_polling_tick
           synchronize do
-            @pollings.delete(thread_id)
+            @pollings.delete(fiber_id)
           end
         end
 
         # Update the processing tick time
         def mark_consumption_tick
           synchronize do
-            @consumptions[thread_id] = monotonic_now
+            @consumptions[fiber_id] = monotonic_now
           end
         end
 
-        # Clear current thread consumption time tracker
+        # Clear current fiber consumption time tracker
         def clear_consumption_tick
           synchronize do
-            @consumptions.delete(thread_id)
+            @consumptions.delete(fiber_id)
           end
         end
 

data/lib/karafka/processing/jobs_queue.rb

@@ -14,10 +14,13 @@ module Karafka
     #   we use a single workers poll that can have granular scheduling.
     class JobsQueue
       include Helpers::ConfigImporter.new(
-        concurrency: %i[concurrency],
         tick_interval: %i[internal tick_interval]
       )
 
+      # Set via writer because of a circular dependency: the queue must exist before the pool
+      # (workers need the queue at construction), but the queue needs the pool for concurrency.
+      attr_writer :pool
+
       # @return [Karafka::Processing::JobsQueue]
       def initialize
         @queue = Queue.new
@@ -56,10 +59,15 @@ module Karafka
       #
       # @param job [Jobs::Base] job that we want to run
       def <<(job)
-        # We do not push the job if the queue is closed as it means that it would anyhow not be
-        # executed
         return if @queue.closed?
 
+        # nil is used by WorkersPool to signal a worker to exit during downscaling.
+        # Passed straight through to the raw queue, bypassing statistics and tracking.
+        unless job
+          @queue << job
+          return
+        end
+
         @mutex.synchronize do
           group = @in_processing[job.group_id]
 
@@ -70,7 +78,7 @@ module Karafka
           # Assume that moving to queue means being picked up immediately not to create stats
           # race conditions because of pop overhead. If there are workers available, we assume
           # work is going to be handled as we never reject enqueued jobs
-          if @statistics[:busy] < concurrency
+          if @statistics[:busy] < @pool.size
             @statistics[:busy] += 1
           else
             # If system is fully loaded, it means this job is indeed enqueued

data/lib/karafka/processing/worker.rb

@@ -27,10 +27,12 @@ module Karafka
       attr_reader :id
 
       # @param jobs_queue [JobsQueue]
+      # @param pool [WorkersPool] pool this worker belongs to (for deregistration on shutdown)
       # @return [Worker]
-      def initialize(jobs_queue)
+      def initialize(jobs_queue, pool)
         @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
+        @pool = pool
         @non_wrapped_flow = worker_job_call_wrapper == false
       end
 
@@ -80,6 +82,9 @@ module Karafka
             end
           end
         else
+          # nil means either queue closed (full shutdown) or pool downscaling.
+          # Either way, deregister from pool so it reflects the actual worker count.
+          @pool.deregister(self)
           false
         end
       # We signal critical exceptions, notify and do not allow worker to fail
@@ -95,7 +100,7 @@ module Karafka
           type: "worker.process.error"
         )
       ensure
-        # job can be nil when the queue is being closed
+        # job can be nil when the queue is being closed or during pool downscaling
         if job
           @jobs_queue.complete(job)
           job.finish!

data/lib/karafka/processing/workers_pool.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Dynamic thread pool that manages worker threads.
+    # Supports scaling at runtime via {#scale}.
+    #
+    # All public methods that read or mutate `@workers` are synchronized via `@mutex`.
+    # `@size` is always updated under `@mutex` but can be read without locking for performance
+    # (integer assignment is atomic in MRI).
+    class WorkersPool
+      include Helpers::ConfigImporter.new(
+        concurrency: %i[concurrency],
+        worker_thread_priority: %i[worker_thread_priority],
+        monitor: %i[monitor]
+      )
+
+      # @return [Integer] current number of workers registered in the pool.
+      #   Reflects the actual thread count, not a target. After a downscale request this value
+      #   converges towards the target as workers pick up nil sentinels and deregister.
+      #   Updated atomically under mutex, safe to read without locking.
+      attr_reader :size
+
+      # Jobs queue reference, set by the Runner after both pool and queue are created.
+      # Must be assigned before calling {#scale}.
+      attr_writer :jobs_queue
+
+      # Initializes an empty pool with zero workers.
+      # Workers are not started until {#scale} is called, allowing the pool to be created early
+      # (e.g. in Server.run) before the jobs queue exists.
+      #
+      # @return [WorkersPool]
+      def initialize
+        @jobs_queue = nil
+        @workers = []
+        @size = 0
+        @mutex = Mutex.new
+        # Monotonically increasing index for naming worker threads. Indices are never reused
+        # after a worker exits, so thread names remain unique across the lifetime of the process
+        # and make it easy to correlate log entries with specific worker generations.
+        @next_index = 0
+      end
+
+      # Scale pool towards `target` workers (minimum 1).
+      #
+      # **Scaling up** is synchronous -- new worker threads are spawned and registered before this
+      # method returns. {#size} reflects the new count immediately.
+      #
+      # **Scaling down** is asynchronous -- nil sentinels are enqueued and workers exit when they
+      # pick one up. {#size} decreases gradually as workers deregister themselves. Callers that
+      # need to know when downsizing is complete should poll {#size} or listen for the
+      # `worker.scaling.down` instrumentation event (whose `:to` payload reports the *target*,
+      # not the current count).
+      #
+      # The entire read-decide-act cycle is synchronized to prevent stale reads.
+      # Instrumentation runs outside the mutex to avoid holding the lock during user callbacks.
+      #
+      # @param target [Integer] desired number of workers
+      def scale(target)
+        raise(Karafka::Errors::BaseError, "jobs_queue must be set before scaling") unless @jobs_queue
+
+        target = [target, 1].max
+        event = nil
+
+        @mutex.synchronize do
+          current = @workers.size
+          delta = target - current
+
+          if delta.positive?
+            event = grow(delta)
+          elsif delta.negative?
+            event = shrink(delta.abs)
+          end
+        end
+
+        return unless event
+
+        monitor.instrument(*event)
+      end
+
+      # @return [Boolean] true if all workers have stopped
+      def stopped?
+        snapshot.none?(&:alive?)
+      end
+
+      # @return [Array<Worker>] workers that are still alive
+      def alive
+        snapshot.select(&:alive?)
+      end
+
+      # Forcefully terminate all worker threads.
+      def terminate
+        snapshot.each(&:terminate)
+      end
+
+      # Wait for all current workers to finish.
+      def join
+        snapshot.each(&:join)
+      end
+
+      # Called by a worker when it exits (queue closed or pool downscaling).
+      # Thread-safe -- worker threads call this from their own thread.
+      #
+      # @param worker [Worker] worker to remove from the pool
+      def deregister(worker)
+        @mutex.synchronize do
+          @workers.delete(worker)
+          @size = @workers.size
+        end
+      end
+
+      private
+
+      # @return [Array<Worker>] snapshot of workers taken under mutex
+      def snapshot
+        @mutex.synchronize { @workers.dup }
+      end
+
+      # Add `count` workers and start their threads immediately.
+      # Must be called under `@mutex` (from {#scale}) or during construction (no contention).
+      #
+      # @param count [Integer] number of workers to add
+      # @return [Array] instrumentation event args to be emitted outside the mutex
+      def grow(count)
+        from = @workers.size
+
+        count.times do
+          worker = Worker.new(@jobs_queue, self)
+          @workers << worker
+          worker.async_call("karafka.worker##{@next_index}", worker_thread_priority)
+          @next_index += 1
+        end
+
+        @size = @workers.size
+
+        ["worker.scaling.up", { workers_pool: self, from: from, to: @size }]
+      end
+
+      # Push nil into the queue to signal workers to exit.
+      # Whichever workers pick them up will deregister and stop.
+      # Must be called under `@mutex` (from {#scale}).
+      #
+      # @param count [Integer] number of workers to remove
+      # @return [Array, nil] instrumentation event args or nil if no-op
+      # @note Never shrinks below 1 worker.
+      def shrink(count)
+        effective = [count, @workers.size - 1].min
+        return if effective <= 0
+
+        from = @workers.size
+        effective.times { @jobs_queue << nil }
+        to = from - effective
+
+        ["worker.scaling.down", { workers_pool: self, from: from, to: to }]
+      end
+    end
+  end
+end

data/lib/karafka/runner.rb

@@ -4,20 +4,20 @@ module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
     include Helpers::ConfigImporter.new(
-      worker_thread_priority: %i[worker_thread_priority],
       manager: %i[internal connection manager],
-      conductor: %i[internal connection conductor],
-      jobs_queue_class: %i[internal processing jobs_queue_class]
+      conductor: %i[internal connection conductor]
     )
 
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
-      # Despite possibility of having several independent listeners, we aim to have one queue for
-      # jobs across and one workers poll for that
-      jobs_queue = jobs_queue_class.new
+      jobs_queue = Karafka::Server.jobs_queue
+      workers = Karafka::Server.workers
+
+      # Wire up the circular dependency between pool and queue
+      workers.jobs_queue = jobs_queue
+      jobs_queue.pool = workers
 
-      workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
 
       # We mark it prior to delegating to the manager as manager will have to start at least one
@@ -27,17 +27,12 @@ module Karafka
       # Register all the listeners so they can be started and managed
       manager.register(listeners)
 
-      workers.each_with_index do |worker, i|
-        worker.async_call(
-          "karafka.worker##{i}",
-          worker_thread_priority
-        )
-      end
-
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
-      Karafka::Server.jobs_queue = jobs_queue
+
+      # Start worker threads after listeners are created so a failure in the boot steps above
+      # does not leave live worker threads blocked on an open queue.
+      workers.scale(Karafka::App.config.concurrency)
 
       until manager.done?
         conductor.wait
@@ -56,7 +51,7 @@ module Karafka
       # with everything. One thing worth keeping in mind though: It is the end user responsibility
       # to handle the shutdown detection in their long-running processes. Otherwise if timeout
       # is exceeded, there will be a forced shutdown.
-      workers.each(&:join)
+      workers.join
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
     rescue => e
@@ -67,6 +62,11 @@ module Karafka
         type: "runner.call.error"
       )
       Karafka::App.stop!
+
+      # Clean up workers so we don't leak threads blocked on the queue
+      jobs_queue.close
+      workers.join
+
       raise e
     end
   end

data/lib/karafka/server.rb

@@ -10,7 +10,8 @@ module Karafka
       shutdown_timeout: %i[shutdown_timeout],
       forceful_exit_code: %i[internal forceful_exit_code],
       forceful_shutdown_wait: %i[internal forceful_shutdown_wait],
-      process: %i[internal process]
+      process: %i[internal process],
+      jobs_queue_class: %i[internal processing jobs_queue_class]
     )
 
     class << self
@@ -39,8 +40,7 @@ module Karafka
 
       # Method which runs app
       def run
-        self.listeners = []
-        self.workers = []
+        prepare
 
         # We need to validate this prior to running because it may be executed also from the
         # embedded
@@ -93,6 +93,7 @@ module Karafka
       # @note We don't need to sleep because Karafka::Runner is locking and waiting to finish loop
       # (and it won't happen until we explicitly want to stop)
       def start
+        prepare
         Karafka::Runner.new.call
       end
 
@@ -117,7 +118,7 @@ module Karafka
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
           all_listeners_stopped = listeners.all?(&:stopped?)
-          all_workers_stopped = workers.none?(&:alive?)
+          all_workers_stopped = workers.stopped?
 
           return if all_listeners_stopped && all_workers_stopped
 
@@ -127,7 +128,7 @@ module Karafka
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
         active_listeners = listeners.select(&:active?)
-        alive_workers = workers.select(&:alive?)
+        alive_workers = workers.alive
 
         # Collect details about subscription groups that still have jobs in processing
         in_processing = jobs_queue ? jobs_queue.in_processing : {}
@@ -143,7 +144,7 @@ module Karafka
         )
 
         # We're done waiting, lets kill them!
-        workers.each(&:terminate)
+        workers.terminate
         listeners.active.each(&:terminate)
 
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
@@ -186,9 +187,26 @@ module Karafka
           WaterDrop::ConnectionPool.close
 
           Karafka::App.terminate!
+
+          # Allow prepare to run again if the server is restarted (e.g. reset_status in tests,
+          # or embedded re-start).
+          @prepared = false
         end
       end
 
+      # Initializes listeners, jobs queue and workers pool.
+      # Called from both {.run} (standalone) and {.start} (embedded). Guarded so it runs only
+      # once even when {.run} delegates to {.start}.
+      def prepare
+        return if @prepared
+
+        @prepared = true
+
+        self.listeners = []
+        self.jobs_queue = jobs_queue_class.new
+        self.workers = Processing::WorkersPool.new
+      end
+
       # Quiets the Karafka server.
       #
       # Karafka will stop processing but won't quit the consumer group, so no rebalance will be

data/lib/karafka/setup/attributes_map.rb

@@ -125,6 +125,7 @@ module Karafka
         sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
+        sasl.oauthbearer.sub.claim.name
         sasl.oauthbearer.token.endpoint.url
         sasl.password
         sasl.username
@@ -283,6 +284,7 @@ module Karafka
         sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
+        sasl.oauthbearer.sub.claim.name
         sasl.oauthbearer.token.endpoint.url
         sasl.password
         sasl.username

data/lib/karafka/setup/contracts/config.rb

@@ -170,6 +170,23 @@ module Karafka
           detected_errors
         end
 
+        # Certain kafka settings are managed internally by Karafka and should not be set
+        # directly. Setting them manually may cause misbehaviours and other unexpected issues.
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          managed_keys = Karafka::Setup::DefaultsInjector.managed_keys
+          detected_errors = []
+
+          data.fetch(:kafka).each_key do |key|
+            next unless managed_keys.include?(key)
+
+            detected_errors << [[:kafka, key], :managed_key_not_supported]
+          end
+
+          detected_errors
+        end
+
         virtual do |data, errors|
           next unless errors.empty?
 

data/lib/karafka/setup/defaults_injector.rb

@@ -52,6 +52,16 @@ module Karafka
       )
 
       class << self
+        # Kafka settings that are managed internally by Karafka and should not be set directly
+        # by users. Setting them manually may cause misbehaviours and other unexpected issues.
+        #
+        # @return [Set<Symbol>] set of managed kafka setting keys
+        def managed_keys
+          @managed_keys ||= Set[
+            :"statistics.unassigned.include"
+          ]
+        end
+
         # Propagates the kafka setting defaults unless they are already present for consumer config
         # This makes it easier to set some values that users usually don't change but still allows
         # them to overwrite the whole hash if they want to

data/lib/karafka/swarm/liveness_listener.rb

@@ -19,6 +19,13 @@ module Karafka
         @mutex = Mutex.new
       end
 
+      # Report liveness before the first fetch loop iteration so the supervisor gets an initial
+      # healthy report even if the first consumption takes longer than the report timeout.
+      # @param _event [Karafka::Core::Monitoring::Event]
+      def on_connection_listener_before_fetch_loop(_event)
+        report_liveness
+      end
+
       # Report from the fetch loop at the top of each iteration
       # @param _event [Karafka::Core::Monitoring::Event]
       def on_connection_listener_fetch_loop(_event)

data/lib/karafka/swarm/manager.rb

@@ -19,12 +19,12 @@ module Karafka
         node_restart_timeout: %i[internal swarm node_restart_timeout]
       )
 
-      # Status we issue when we decide to shutdown unresponsive node
+      # Status we issue when we decide to shutdown a non-reporting node
       # We use -1 because nodes are expected to report 0+ statuses and we can use negative numbers
       # for non-node based statuses
-      NOT_RESPONDING_SHUTDOWN_STATUS = -1
+      NOT_REPORTING_SHUTDOWN_STATUS = -1
 
-      private_constant :NOT_RESPONDING_SHUTDOWN_STATUS
+      private_constant :NOT_REPORTING_SHUTDOWN_STATUS
 
       # @return [Array<Node>] All nodes that manager manages
       attr_reader :nodes
@@ -84,7 +84,7 @@ module Karafka
             if node.alive?
               next if terminate_if_hanging(statuses, node)
               next if stop_if_not_healthy(statuses, node)
-              next if stop_if_not_responding(statuses, node)
+              next if stop_if_not_reporting(statuses, node)
             else
               next if cleanup_one(statuses, node)
               next if restart_after_timeout(statuses, node)
@@ -144,12 +144,12 @@ module Karafka
         end
       end
 
-      # If node stopped responding, starts the stopping procedure.
+      # If node stopped reporting, starts the stopping procedure.
       #
       # @param statuses [Hash] hash with statuses transitions with times
       # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
-      def stop_if_not_responding(statuses, node)
+      def stop_if_not_reporting(statuses, node)
         # Do nothing if already stopping
         return true if statuses.key?(:stop)
         # Do nothing if we've received status update recently enough
@@ -160,7 +160,7 @@ module Karafka
           "swarm.manager.stopping",
           caller: self,
           node: node,
-          status: NOT_RESPONDING_SHUTDOWN_STATUS
+          status: NOT_REPORTING_SHUTDOWN_STATUS
         ) do
           node.stop
           statuses[:stop] = monotonic_now

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = "2.5.9"
+  VERSION = "2.5.10.rc1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.9
+  version: 2.5.10.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -15,7 +15,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.6
+        version: 2.5.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
@@ -25,7 +25,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.6
+        version: 2.5.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
@@ -35,14 +35,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.24.0
+        version: 0.26.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.24.0
+        version: 0.26.1
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
@@ -456,6 +456,7 @@ files:
 - lib/karafka/pro/scheduled_messages/setup/config.rb
 - lib/karafka/pro/scheduled_messages/state.rb
 - lib/karafka/pro/scheduled_messages/tracker.rb
+- lib/karafka/pro/setup/defaults_injector.rb
 - lib/karafka/pro/swarm/liveness_listener.rb
 - lib/karafka/process.rb
 - lib/karafka/processing/coordinator.rb
@@ -486,7 +487,7 @@ files:
 - lib/karafka/processing/strategies/mom.rb
 - lib/karafka/processing/strategy_selector.rb
 - lib/karafka/processing/worker.rb
-- lib/karafka/processing/workers_batch.rb
+- lib/karafka/processing/workers_pool.rb
 - lib/karafka/railtie.rb
 - lib/karafka/routing/activity_manager.rb
 - lib/karafka/routing/builder.rb

data/lib/karafka/processing/workers_batch.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Abstraction layer around workers batch.
-    class WorkersBatch
-      include Enumerable
-      include Helpers::ConfigImporter.new(
-        concurrency: %i[concurrency]
-      )
-
-      # @param jobs_queue [JobsQueue]
-      # @return [WorkersBatch]
-      def initialize(jobs_queue)
-        @batch = Array.new(concurrency) { Processing::Worker.new(jobs_queue) }
-      end
-
-      # Iterates over available workers and yields each worker
-      def each(&)
-        @batch.each(&)
-      end
-
-      # @return [Integer] number of workers in the batch
-      def size
-        @batch.size
-      end
-    end
-  end
-end