karafka 2.5.6 → 2.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11edff86c8615652130786431e179d242dcc33130e5aabbbf5e0f5ed6d4138fe
-  data.tar.gz: 69244021709283a153da19907a424c230a3597832beec6acbb39dbc02e738256
+  metadata.gz: b395542efd1d57ac4f9fd89091866a39c23e2c48d446bb20fb3628e7952f762e
+  data.tar.gz: 14aeb17e690a257bb0d8d7e57b45e797da19adf22f2e5873bd88e18a9c0a6aea
 SHA512:
-  metadata.gz: 843548637c77ace03cde5c1f4150226c244b1b3343c6acd17d16c19cdc798c6d770611522eed7ebef76830f3e551c75afa176f2937c1fd63045e66ccb6276701
-  data.tar.gz: a0696c28b998c8e13d4b0e0d3f4252245f27e6f93e0d75d2153a93cc1dcc53b45fbe8416ba0a9e6025753ec5d107088d578f03d0a4cada3bb5ed2eb4e5341518
+  metadata.gz: e9003ff1411da366cc3f70a77e7b949bdda7fed9d717eb6a2c151e3844af1fb0cdfca9f4c220be8d4a692b750e99c202d2afa05e57ffbac7c274579d065ee54c
+  data.tar.gz: 4807ec1df8ed8f8169e6f38b9ce5b47e541485e4f4d551822bd2ea9d578219ee565ca71b27d66a9aeef55917ffa2d16fddd629371f48661afd94271eff9c228d

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Karafka Framework Changelog
 
+## 2.5.7 (2026-03-16)
+- [Enhancement] Report detailed blocking information (active listeners, alive workers, and in-processing jobs) during forceful shutdown instead of only aggregate counts.
+- [Enhancement] Improve `ForcefulShutdownError` description to clearly explain when and why it is raised.
+- [Enhancement] Cache `messages.last` in `BatchMetadata` builder to avoid duplicate array traversal.
+- [Enhancement] Optimize `VirtualOffsetManager#mark` to use a single array scan instead of separate `include?` and `index` calls (Pro).
+- [Enhancement] Optimize `VirtualOffsetManager#materialize_real_offset` to use `keys.sort` instead of `to_a.sort_by` with tuple destructuring (Pro).
+- [Enhancement] Optimize `IntervalRunner#call` to use a single `monotonic_now` call instead of two per invocation.
+- [Enhancement] Support WaterDrop `:fd` mode in Swarm.
+- [Maintenance] Use both `:fd` and `:thread` producer backends in CI.
+- [Maintenance] Include spec file hash in integration test topic names for easier traceability in Kafka logs (#3056).
+- [Fix] Remove duplicate topic creation in multi-broker health integration specs (#3056).
+- [Fix] Preserve producer-specific kafka settings (e.g., `enable.idempotence`) when recreating the producer in swarm forks.
+
 ## 2.5.6 (2026-02-28)
 - **[Feature]** Add `karafka topics health` command to check Kafka topics for replication and durability issues, detecting no redundancy (RF=1), zero fault tolerance (RF≤min.insync), and low durability (min.insync=1) configurations with color-coded severity grouping and actionable recommendations (Pro).
 - [Enhancement] Optimize license loading process by reading license files directly from the gem directory instead of requiring the entire gem, reducing initialization overhead and adding support for user-defined License modules.

data/lib/karafka/errors.rb

@@ -51,7 +51,9 @@ module Karafka
       end
     end
 
-    # Raised when we've waited enough for shutting down a non-responsive process
+    # Raised when the graceful shutdown timeout has been exceeded and Karafka must forcefully
+    # terminate remaining listeners and workers. This typically happens when consumer processing
+    # or shutdown jobs take longer than the configured `shutdown_timeout`.
     ForcefulShutdownError = Class.new(BaseError)
 
     # Raised when the jobs queue receives a job that should not be received as it would cause

data/lib/karafka/helpers/interval_runner.rb

@@ -26,9 +26,11 @@ module Karafka
 
       # Runs the requested code if it was not executed previously recently
       def call
-        return if monotonic_now - @last_called_at < @interval
+        now = monotonic_now
 
-        @last_called_at = monotonic_now
+        return if now - @last_called_at < @interval
+
+        @last_called_at = now
 
         @block.call
       end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -385,21 +385,34 @@ module Karafka
           fatal "Runner crashed due to an error: #{details}"
           fatal backtrace
         when "app.stopping.error"
-          # Counts number of workers and listeners that were still active when forcing the
-          # shutdown. Please note, that unless all listeners are closed, workers will not finalize
-          # their operations as well.
-          # We need to check if listeners and workers are assigned as during super early stages of
-          # boot they are not.
-          listeners = Server.listeners ? Server.listeners.count(&:active?) : 0
-          workers = Server.workers ? Server.workers.count(&:alive?) : 0
+          active_listeners = event.payload[:active_listeners]
+          alive_workers = event.payload[:alive_workers]
+          in_processing = event.payload[:in_processing]
 
           message = <<~MSG.tr("\n", " ").strip!
             Forceful Karafka server stop with:
-            #{workers} active workers and
-            #{listeners} active listeners
+            #{alive_workers.size} active workers and
+            #{active_listeners.size} active listeners
           MSG
 
           error message
+
+          active_listeners.each do |listener|
+            error "Listener #{listener.id} for #{listener.subscription_group.name} still active"
+          end
+
+          in_processing.each do |group_id, jobs|
+            next if jobs.empty?
+
+            jobs.each do |job|
+              job_class = job.class.name.split("::").last
+              topic_name = job.executor.topic.name
+              partition = job.executor.partition
+
+              error "In processing: #{job_class} job for #{topic_name}/#{partition} " \
+                    "(group: #{group_id})"
+            end
+          end
         when "app.forceful_stopping.error"
           error "Forceful shutdown error occurred: #{details}"
           error backtrace

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -117,7 +117,12 @@ module Karafka
             when "runner.call.error"
               fatal "Runner crashed due to an error: #{error}"
             when "app.stopping.error"
-              error "Forceful Karafka server stop"
+              active_listeners = event.payload[:active_listeners]
+              alive_workers = event.payload[:alive_workers]
+
+              error "Forceful Karafka server stop with: " \
+                    "#{alive_workers.size} active workers and " \
+                    "#{active_listeners.size} active listeners"
             when "app.forceful_stopping.error"
               error "Forceful shutdown error occurred: #{error}"
             when "librdkafka.error"

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -17,16 +17,18 @@ module Karafka
           # @note We do not set `processed_at` as this needs to be assigned when the batch is
           #   picked up for processing.
           def call(messages, topic, partition, scheduled_at)
+            last_message = messages.last
+
             Karafka::Messages::BatchMetadata.new(
               size: messages.size,
               first_offset: messages.first&.offset || -1001,
-              last_offset: messages.last&.offset || -1001,
+              last_offset: last_message&.offset || -1001,
               deserializers: topic.deserializers,
               partition: partition,
               topic: topic.name,
               # We go with the assumption that the creation of the whole batch is the last message
               # creation time
-              created_at: local_created_at(messages.last),
+              created_at: local_created_at(last_message),
               # When this batch was built and scheduled for execution
               scheduled_at: scheduled_at,
               # This needs to be set to a correct value prior to processing starting

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -91,17 +91,27 @@ module Karafka
             @offsets_metadata[offset] = offset_metadata
             @current_offset_metadata = offset_metadata
 
-            group = @groups.find { |reg_group| reg_group.include?(offset) }
+            group = nil
+            position = nil
+
+            @groups.each do |reg_group|
+              pos = reg_group.index(offset)
+
+              if pos
+                group = reg_group
+                position = pos
+                break
+              end
+            end
 
             # This case can happen when someone uses MoM and wants to mark message from a previous
             # batch as consumed. We can add it, since the real offset refresh will point to it
             unless group
               group = [offset]
+              position = 0
               @groups << group
             end
 
-            position = group.index(offset)
-
             # Mark all previous messages from the same group also as virtually consumed
             group[0..position].each do |markable_offset|
               # Set previous messages metadata offset as the offset of higher one for overwrites
@@ -135,7 +145,7 @@ module Karafka
 
           # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
           def marked
-            @marked.select { |_, status| status }.map(&:first).sort
+            @marked.select { |_, status| status }.map { |offset, _| offset }.sort
           end
 
           # Is there a real offset we can mark as consumed
@@ -171,11 +181,11 @@ module Karafka
           private
 
           # Recomputes the biggest possible real offset we can have.
-          # It picks the the biggest offset that has uninterrupted stream of virtually marked as
+          # It picks the biggest offset that has uninterrupted stream of virtually marked as
           # consumed because this will be the collective offset.
           def materialize_real_offset
-            @marked.to_a.sort_by(&:first).each do |offset, marked|
-              break unless marked
+            @marked.keys.sort.each do |offset|
+              break unless @marked[offset]
 
               @real_offset = offset
             end

data/lib/karafka/processing/jobs_queue.rb

@@ -180,6 +180,16 @@ module Karafka
         end
       end
 
+      # Returns a snapshot of all jobs currently in processing per group.
+      # Useful for diagnostics during forceful shutdown to understand what is blocking.
+      #
+      # @return [Hash{String => Array<Jobs::Base>}] hash mapping group ids to arrays of jobs
+      def in_processing
+        @mutex.synchronize do
+          @in_processing.transform_values(&:dup).freeze
+        end
+      end
+
       private
 
       # @param group_id [String] id of the group in which jobs we're interested.

data/lib/karafka/server.rb

@@ -126,10 +126,19 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
+        active_listeners = listeners.select(&:active?)
+        alive_workers = workers.select(&:alive?)
+
+        # Collect details about subscription groups that still have jobs in processing
+        in_processing = jobs_queue ? jobs_queue.in_processing : {}
+
         Karafka.monitor.instrument(
           "error.occurred",
           caller: self,
           error: e,
+          active_listeners: active_listeners,
+          alive_workers: alive_workers,
+          in_processing: in_processing,
           type: "app.stopping.error"
         )
 

data/lib/karafka/swarm/node.rb

@@ -27,18 +27,6 @@ module Karafka
       # @return [Integer] pid of the node
       attr_reader :pid
 
-      # When re-creating a producer in the fork, those are not attributes we want to inherit
-      # from the parent process because they are updated in the fork. If user wants to take those
-      # from the parent process, he should redefine them by overwriting the whole producer.
-      SKIPPABLE_NEW_PRODUCER_ATTRIBUTES = %i[
-        id
-        kafka
-        logger
-        oauth
-      ].freeze
-
-      private_constant :SKIPPABLE_NEW_PRODUCER_ATTRIBUTES
-
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
@@ -70,24 +58,7 @@ module Karafka
           config.producer.close
 
           old_producer = config.producer
-          old_producer_config = old_producer.config
-
-          # Supervisor producer is closed, hence we need a new one here
-          config.producer = WaterDrop::Producer.new do |p_config|
-            p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
-            p_config.logger = config.logger
-
-            old_producer_config.to_h.each do |key, value|
-              next if SKIPPABLE_NEW_PRODUCER_ATTRIBUTES.include?(key)
-
-              p_config.public_send("#{key}=", value)
-            end
-
-            # Namespaced attributes need to be migrated directly on their config node
-            old_producer_config.oauth.to_h.each do |key, value|
-              p_config.oauth.public_send("#{key}=", value)
-            end
-          end
+          config.producer = ProducerReplacer.new.call(old_producer, kafka, config.logger)
 
           @pid = ::Process.pid
           @reader.close

data/lib/karafka/swarm/producer_replacer.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Builds a new WaterDrop producer that inherits configuration from an old one
+    #
+    # When a swarm node forks, the parent's producer must be replaced with a new one.
+    # This class encapsulates the logic for building that new producer, inheriting all relevant
+    # settings from the old one while generating fresh connection-level configuration.
+    class ProducerReplacer
+      # Attributes that should not be directly copied from the old producer config because they
+      # are either regenerated fresh (kafka, logger, id) or handled via their own namespaced
+      # migration (oauth, polling, polling.fd).
+      SKIPPABLE_ATTRIBUTES = %i[
+        id
+        kafka
+        logger
+        oauth
+        polling
+        fd
+      ].freeze
+
+      private_constant :SKIPPABLE_ATTRIBUTES
+
+      # Builds a new WaterDrop producer inheriting configuration from the old one
+      #
+      # @param old_producer [WaterDrop::Producer] the old producer to inherit settings from
+      # @param kafka [Hash] app-level kafka configuration
+      # @param logger [Object] logger instance for the new producer
+      # @return [WaterDrop::Producer] new producer with inherited configuration
+      def call(old_producer, kafka, logger)
+        old_producer_config = old_producer.config
+
+        WaterDrop::Producer.new do |p_config|
+          p_config.logger = logger
+
+          migrate_kafka(p_config, old_producer_config, kafka)
+          migrate_root(p_config, old_producer_config)
+          migrate_oauth(p_config, old_producer_config)
+          migrate_polling(p_config, old_producer_config)
+          migrate_polling_fd(p_config, old_producer_config)
+        end
+      end
+
+      private
+
+      # Migrates root-level producer attributes from the old producer, skipping those that are
+      # regenerated fresh or handled by their own namespaced migration
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_root(p_config, old_producer_config)
+        old_producer_config.to_h.each do |key, value|
+          next if SKIPPABLE_ATTRIBUTES.include?(key)
+
+          p_config.public_send("#{key}=", value)
+        end
+      end
+
+      # Builds fresh kafka config from app-level settings and preserves any producer-specific
+      # kafka settings from the old producer (e.g., enable.idempotence) that aren't in the
+      # base app kafka config
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      # @param kafka [Hash] app-level kafka configuration
+      def migrate_kafka(p_config, old_producer_config, kafka)
+        p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
+
+        old_producer_config.kafka.each do |key, value|
+          next if p_config.kafka.key?(key)
+
+          p_config.kafka[key] = value
+        end
+      end
+
+      # Migrates oauth configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_oauth(p_config, old_producer_config)
+        old_producer_config.oauth.to_h.each do |key, value|
+          p_config.oauth.public_send("#{key}=", value)
+        end
+      end
+
+      # Migrates polling configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_polling(p_config, old_producer_config)
+        old_producer_config.polling.to_h.each do |key, value|
+          next if SKIPPABLE_ATTRIBUTES.include?(key)
+
+          p_config.polling.public_send("#{key}=", value)
+        end
+      end
+
+      # Migrates polling fd configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_polling_fd(p_config, old_producer_config)
+        old_producer_config.polling.fd.to_h.each do |key, value|
+          p_config.polling.fd.public_send("#{key}=", value)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/supervisor.rb

@@ -152,6 +152,9 @@ module Karafka
           caller: self,
           error: e,
           manager: manager,
+          active_listeners: [],
+          alive_workers: [],
+          in_processing: {},
           type: "app.stopping.error"
         )
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = "2.5.6"
+  VERSION = "2.5.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.6
+  version: 2.5.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -542,6 +542,7 @@ files:
 - lib/karafka/swarm/liveness_listener.rb
 - lib/karafka/swarm/manager.rb
 - lib/karafka/swarm/node.rb
+- lib/karafka/swarm/producer_replacer.rb
 - lib/karafka/swarm/supervisor.rb
 - lib/karafka/templates/application_consumer.rb.erb
 - lib/karafka/templates/example_consumer.rb.erb