karafka 2.6.0.beta1 → 2.6.0.rc1

data/lib/karafka/processing/consumer_groups/jobs/eofed.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Consumer-group-specific job types
       module Jobs
@@ -11,7 +9,8 @@ module Karafka
         class Eofed < ::Karafka::Processing::Jobs::Base
           self.action = :eofed
 
-          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run the job
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is
+          #   supposed to run the job
           # @return [Eofed]
           def initialize(executor)
             @executor = executor

data/lib/karafka/processing/consumer_groups/jobs/idle.rb

@@ -9,8 +9,8 @@ module Karafka
         class Idle < ::Karafka::Processing::Jobs::Base
           self.action = :idle
 
-          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run a given
-          #   job on an active consumer
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is
+          #   supposed to run a given job on an active consumer
           # @return [Idle]
           def initialize(executor)
             @executor = executor

data/lib/karafka/processing/consumer_groups/jobs/revoked.rb

@@ -2,16 +2,14 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
-      # Consumer-group-specific job types
       module Jobs
         # Job that runs the revoked operation when we lose a partition on a consumer that lost it.
         class Revoked < ::Karafka::Processing::Jobs::Base
           self.action = :revoked
 
-          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run the job
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is
+          #   supposed to run the job
           # @return [Revoked]
           def initialize(executor)
             @executor = executor

data/lib/karafka/processing/consumer_groups/jobs/shutdown.rb

@@ -8,8 +8,8 @@ module Karafka
         class Shutdown < ::Karafka::Processing::Jobs::Base
           self.action = :shutdown
 
-          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is supposed to run a given
-          #   job on an active consumer
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is
+          #   supposed to run a given job on an active consumer
           # @return [Shutdown]
           def initialize(executor)
             @executor = executor

data/lib/karafka/processing/consumer_groups/partitioner.rb

@@ -2,11 +2,8 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
-      # Basic partitioner for work division
-      # It does not divide any work.
+      # Basic partitioner for work division. It does not divide any work.
       class Partitioner
         # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
         def initialize(subscription_group)
@@ -15,8 +12,8 @@ module Karafka
 
         # @param _topic [String] topic name
         # @param messages [Array<Karafka::Messages::Message>] karafka messages
-        # @param _coordinator [Karafka::Processing::ConsumerGroups::Coordinator] processing coordinator that will
-        #   be used with those messages
+        # @param _coordinator [Karafka::Processing::ConsumerGroups::Coordinator] processing
+        #   coordinator that will be used with those messages
         # @yieldparam [Integer] group id
         # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
         def call(_topic, messages, _coordinator)

data/lib/karafka/processing/consumer_groups/strategies/aj_dlq_mom.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
         # ActiveJob strategy to cooperate with the DLQ.
@@ -28,6 +26,11 @@ module Karafka
             if coordinator.success?
               # Do NOT commit offsets, they are comitted after each job in the AJ consumer.
               coordinator.pause_tracker.reset
+            # Process-critical errors are never dispatched to the DLQ regardless of the retries
+            # state: the retry pause protects the partition during the critical shutdown and
+            # the failed batch is redelivered after the restart
+            elsif critical_error?(coordinator.consumption(self).cause)
+              retry_after_pause
             elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
               retry_after_pause
             else

data/lib/karafka/processing/consumer_groups/strategies/aj_mom.rb

@@ -2,12 +2,10 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
-        # ActiveJob enabled
-        # Manual offset management enabled
+        # - ActiveJob enabled
+        # - Manual offset management enabled
         #
         # This is the default AJ strategy since AJ cannot be used without MOM
         module AjMom

data/lib/karafka/processing/consumer_groups/strategies/base.rb

@@ -2,12 +2,10 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
-      # Our processing patterns differ depending on various features configurations
-      # In this namespace we collect strategies for particular feature combinations to simplify the
-      # design. Based on features combinations we can then select handling strategy for a given case.
+      # Our processing patterns differ depending on various features configurations. In this
+      # namespace we collect strategies for particular feature combinations to simplify the design.
+      # Based on features combinations we can then select handling strategy for a given case.
       #
       # @note The lack of common code here is intentional. It would get complex if there would be
       #   any type of composition, so each strategy is expected to be self-sufficient

data/lib/karafka/processing/consumer_groups/strategies/default.rb

@@ -2,13 +2,12 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
-        # No features enabled.
-        # No manual offset management
-        # No long running jobs
+        # No features enabled:
+        # - No manual offset management
+        # - No long running jobs
+        #
         # Nothing. Just standard, automatic flow
         module Default
           include Base
@@ -59,9 +58,18 @@ module Karafka
             # In case like this we ignore marking
             return true if seek_offset.nil?
             # Ignore double markings of the same offset
+            # Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
+            # allowed and rewinds the seek offset for reprocessing (see #2432)
             return true if (seek_offset - 1) == message.offset
             return false if revoked?
-            return revoked? unless client.mark_as_consumed(message)
+            unless client.mark_as_consumed(message)
+              # Evaluate ownership for its coordinator-revoking side effect - all marking
+              # failure causes are ownership related. The failure itself always reports false:
+              # the offset was not stored, regardless of the current ownership state
+              revoked?
+
+              return false
+            end
 
             self.seek_offset = message.offset + 1
 
@@ -78,10 +86,19 @@ module Karafka
             # In case like this we ignore marking
             return true if seek_offset.nil?
             # Ignore double markings of the same offset
+            # Only this exact re-mark is skipped - marking genuinely older offsets is intentionally
+            # allowed and rewinds the seek offset for reprocessing (see #2432)
             return true if (seek_offset - 1) == message.offset
             return false if revoked?
 
-            return revoked? unless client.mark_as_consumed!(message)
+            unless client.mark_as_consumed!(message)
+              # Evaluate ownership for its coordinator-revoking side effect - all marking
+              # failure causes are ownership related. The failure itself always reports false:
+              # the offset was not committed, regardless of the current ownership state
+              revoked?
+
+              return false
+            end
 
             self.seek_offset = message.offset + 1
 
@@ -91,17 +108,27 @@ module Karafka
           # Triggers an async offset commit
           #
           # @param async [Boolean] should we use async (default) or sync commit
-          # @return [Boolean] true if we still own the partition.
+          # @return [Boolean] true if the commit went through and we still owned the partition
+          #   at that time, false otherwise. False means the offsets were not committed - the
+          #   caller can retry (the offsets stay stored client-side) or treat the partition as
+          #   fenced.
           # @note Due to its async nature, this may not fully represent the offset state in some
           #   edge cases (like for example going beyond max.poll.interval)
+          # @note A commit with no offsets pending is a client-side no-op reported as success
+          #   without broker validation - the marking result is the fence for that case.
           def commit_offsets(async: true)
             # Do not commit if we already lost the assignment
             return false if revoked?
             return true if client.commit_offsets(async: async)
 
-            # This will once more check the librdkafka revocation status and will revoke the
-            # coordinator in case it was not revoked
+            # Evaluate ownership once more for its coordinator-revoking side effect: all
+            # commit failure causes the client reports as false are ownership related. The
+            # failed commit itself always reports false - its value must not depend on the
+            # revocation state, which may not yet be locally visible (or may indicate a
+            # retained partition under a cooperative rebalance generation bump)
             revoked?
+
+            false
           end
 
           # Triggers a synchronous offsets commit to Kafka
@@ -138,7 +165,10 @@ module Karafka
 
             # Mark job as successful
             coordinator.success!(self)
-          rescue => e
+          # Failure recording must be class-agnostic: an unrecorded non-StandardError would
+          # leave the consumption result in its default (successful) state and the after-consume
+          # flow would mark the failed batch as consumed instead of engaging the retry
+          rescue Exception => e
             coordinator.failure!(self, e)
 
             # Re-raise so reported in the consumer
@@ -187,8 +217,8 @@ module Karafka
           end
 
           # We need to always un-pause the processing in case we have lost a given partition.
-          # Otherwise the underlying librdkafka would not know we may want to continue processing and
-          # the pause could in theory last forever
+          # Otherwise the underlying librdkafka would not know we may want to continue processing
+          # and the pause could in theory last forever
           def handle_revoked
             resume
 

data/lib/karafka/processing/consumer_groups/strategies/dlq.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
         # When using dead letter queue, processing won't stop after defined number of retries
@@ -64,6 +62,11 @@ module Karafka
               return if coordinator.manual_pause?
 
               mark_as_consumed(messages.last)
+            # Process-critical errors are never dispatched to the DLQ regardless of the retries
+            # state: the retry pause protects the partition during the critical shutdown and
+            # the failed batch is redelivered after the restart
+            elsif critical_error?(coordinator.consumption(self).cause)
+              retry_after_pause
             elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
               retry_after_pause
             # If we've reached number of retries that we could, we need to skip the first message

data/lib/karafka/processing/consumer_groups/strategies/dlq_mom.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
         # Same as pure dead letter queue but we do not marked failed message as consumed
@@ -23,6 +21,11 @@ module Karafka
 
             if coordinator.success?
               coordinator.pause_tracker.reset
+            # Process-critical errors are never dispatched to the DLQ regardless of the retries
+            # state: the retry pause protects the partition during the critical shutdown and
+            # the failed batch is redelivered after the restart
+            elsif critical_error?(coordinator.consumption(self).cause)
+              retry_after_pause
             elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
               retry_after_pause
             # If we've reached number of retries that we could, we need to skip the first message

data/lib/karafka/processing/consumer_groups/strategies/mom.rb

@@ -2,11 +2,10 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       module Strategies
-        # When using manual offset management, we do not mark as consumed after successful processing
+        # When using manual offset management, we do not mark as consumed after successful
+        # processing
         module Mom
           include Default
 

data/lib/karafka/processing/consumer_groups/strategy_selector.rb

@@ -2,8 +2,6 @@
 
 module Karafka
   module Processing
-    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
-    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
     module ConsumerGroups
       # Selector of appropriate processing strategy matching topic combinations
       class StrategySelector

data/lib/karafka/processing/coordinator.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for karafka-web gem that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::Coordinator

data/lib/karafka/processing/coordinators_buffer.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for karafka-testing gem that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::CoordinatorsBuffer

data/lib/karafka/processing/partitioner.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for external code that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::Partitioner

data/lib/karafka/processing/strategy_selector.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-# Backwards-compatible alias kept for external code that references the old, un-namespaced
-# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
     # @see ConsumerGroups::StrategySelector

data/lib/karafka/processing/worker.rb

@@ -45,10 +45,16 @@ module Karafka
 
       # Fetches a single job, processes it and marks as completed.
       #
-      # @note We do not have error handling here, as no errors should propagate this far. If they
-      #   do, it is a critical error and should bubble up.
+      # @note No user or framework errors should propagate this far. If they do, it is a critical
+      #   error that we instrument while keeping the worker alive.
       #
       # @note Upon closing the jobs queue, worker will close it's thread
+      #
+      # @return [Boolean] should the worker keep running. False only when the jobs queue is
+      #   closed (full shutdown) or a downscale sentinel was received. This decision is
+      #   deliberately decoupled from the processing flow result: the flow passes through the
+      #   user-redefinable `#wrap`, whose return value (or an error raised in user code or in an
+      #   instrumentation subscriber) must never be able to silently kill the worker thread.
       def process
         job = @jobs_queue.pop
 
@@ -76,10 +82,10 @@ module Karafka
               end
 
               job.after_call
-
-              true
             end
           end
+
+          true
         else
           # nil means either queue closed (full shutdown) or pool downscaling.
           # Either way, deregister from pool so it reflects the actual worker count.
@@ -87,9 +93,7 @@ module Karafka
           false
         end
       # We signal critical exceptions, notify and do not allow worker to fail
-      # rubocop:disable Lint/RescueException
       rescue Exception => e
-        # rubocop:enable Lint/RescueException
         monitor.instrument(
           "error.occurred",
           caller: self,
@@ -98,6 +102,10 @@ module Karafka
           error: e,
           type: "worker.process.error"
         )
+
+        # Keep the worker alive. Without an explicit true this arm would return the result of
+        # the instrumentation call (nil) and the worker thread would silently exit
+        true
       ensure
         # job can be nil when the queue is being closed or during pool downscaling
         if job

data/lib/karafka/processing/workers_pool.rb

@@ -39,6 +39,13 @@ module Karafka
         # after a worker exits, so thread names remain unique across the lifetime of the process
         # and make it easy to correlate log entries with specific worker generations.
         @next_index = 0
+        # Number of downscale sentinels enqueued but not yet picked up by a worker. Workers
+        # deregister asynchronously, so the live thread count alone overstates the pool between
+        # a scale-down request and its completion. All sizing decisions use the committed size
+        # (live workers minus pending shrinks) so repeated or overlapping scale-down requests do
+        # not enqueue duplicate sentinels and drain the pool below the requested target (or
+        # below the minimum of one worker).
+        @pending_shrinks = 0
       end
 
       # Scale pool towards `target` workers (minimum 1).
@@ -63,7 +70,9 @@ module Karafka
         event = nil
 
         @mutex.synchronize do
-          current = @workers.size
+          # Compute against the committed size, not the live thread count: sentinels already in
+          # flight are removals this pool is committed to, just not yet executed by the workers
+          current = @workers.size - @pending_shrinks
           delta = target - current
 
           if delta.positive?
@@ -106,6 +115,11 @@ module Karafka
         @mutex.synchronize do
           @workers.delete(worker)
           @size = @workers.size
+          # A worker exits either because it picked up a downscale sentinel or because the jobs
+          # queue was closed for shutdown. The worker cannot tell those apart, so pending
+          # removals are settled here, with a floor because shutdown exits may have no pending
+          # shrink to consume
+          @pending_shrinks -= 1 if @pending_shrinks.positive?
         end
       end
 
@@ -142,14 +156,17 @@ module Karafka
       #
       # @param count [Integer] number of workers to remove
       # @return [Array, nil] instrumentation event args or nil if no-op
-      # @note Never shrinks below 1 worker.
+      # @note Never shrinks below 1 worker, including removals already in flight - the cap is
+      #   computed against the committed size, not the live thread count.
       def shrink(count)
-        effective = [count, @workers.size - 1].min
+        committed = @workers.size - @pending_shrinks
+        effective = [count, committed - 1].min
         return if effective <= 0
 
         from = @workers.size
+        @pending_shrinks += effective
         effective.times { @jobs_queue << nil }
-        to = from - effective
+        to = committed - effective
 
         ["worker.scaling.down", { workers_pool: self, from: from, to: to }]
       end

data/lib/karafka/railtie.rb

@@ -3,7 +3,6 @@
 # This file contains Railtie for auto-configuration
 
 unless Karafka.rails?
-  # Without defining this in any way, Zeitwerk ain't happy so we do it that way
   module Karafka
     class Railtie
     end

data/lib/karafka/routing/contracts/routing.rb

@@ -12,8 +12,8 @@ module Karafka
         end
 
         # Ensures, that when declarative topics strict requirement is on, all topics have
-        # declarative definition (including DLQ topics)
-        # @note It will ignore routing pattern topics because those topics are virtual
+        # declarative definition (including DLQ topics). It will ignore routing pattern topics
+        # because those topics are virtual
         virtual do |data, errors|
           next unless errors.empty?
           # Do not validate declaratives unless required and explicitly enabled

data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb

@@ -7,11 +7,12 @@ module Karafka
         class ActiveJob < Base
           # Topic extensions to be able to check if given topic is ActiveJob topic
           #
-          # @note ActiveJob topics do not have per-topic deserializer configuration. The deserializer
-          #   is configured globally via `config.internal.active_job.deserializer` because Rails
-          #   serializes jobs before dispatching them, requiring a consistent serialization format
-          #   across all ActiveJob topics. If you need custom serialization (e.g., Avro, Protobuf),
-          #   configure it once at the application level rather than per-topic.
+          # @note ActiveJob topics do not have per-topic deserializer configuration. The
+          #   deserializer is configured globally via `config.internal.active_job.deserializer`
+          #   because Rails serializes jobs before dispatching them, requiring a consistent
+          #   serialization format across all ActiveJob topics. If you need custom serialization
+          #   (e.g., Avro, Protobuf), configure it once at the application level rather than
+          #   per-topic.
           module Topic
             # This method sets up the extra instance variable to nil before calling
             # the parent class initializer. The explicit initialization

data/lib/karafka/routing/features/consumer_groups/manual_offset_management/topic.rb

@@ -16,8 +16,8 @@ module Karafka
               super
             end
 
-            # @param active [Boolean] should we stop managing the offset in Karafka and make the user
-            #   responsible for marking messages as consumed.
+            # @param active [Boolean] should we stop managing the offset in Karafka and make the
+            #   user responsible for marking messages as consumed.
             # @return [Config] defined config
             #
             # @note Since this feature supports only one setting (active), we can use the old API

data/lib/karafka/routing/features/deserializers/topic.rb

@@ -3,7 +3,6 @@
 module Karafka
   module Routing
     module Features
-      # Deserializers for all the message details (payload, headers, key)
       class Deserializers < Base
         # Routing topic deserializers API. It allows to configure deserializers for various
         # components of each message.

data/lib/karafka/routing/subscription_group.rb

@@ -16,7 +16,7 @@ module Karafka
 
       attr_reader :id, :name, :topics, :kafka, :group, :position
 
-      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # Backwards compatible alias for `#group`. Kept purely for compatibility - this is an
       # unconditional alias and performs no type validation, so callers should prefer `#group`
       # once additional group types (e.g. KIP-932 share groups) land.
       alias_method :consumer_group, :group

data/lib/karafka/routing/topic.rb

@@ -14,7 +14,7 @@ module Karafka
       attr_reader :id, :name, :group
       attr_writer :consumer
 
-      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # Backwards compatible alias for `#group`. Kept purely for compatibility - this is an
       # unconditional alias and performs no type validation, so callers should prefer `#group`
       # once additional group types (e.g. KIP-932 share groups) land.
       alias_method :consumer_group, :group
@@ -47,9 +47,9 @@ module Karafka
         @group = group
         @attributes = {}
         @active = true
-        # @note We use identifier related to the group that owns a topic, because from Karafka 0.6
-        #   we can handle multiple Kafka instances with the same process and we can have same
-        #   topic name across multiple groups
+        # We use identifier related to the group that owns a topic, because from Karafka 0.6 we can
+        # handle multiple Kafka instances with the same process and we can have same topic name
+        # across multiple groups
         @id = "#{group.id}_#{@name}"
         @consumer = nil
         @active_assigned = false

data/lib/karafka/routing/topics.rb

@@ -7,13 +7,33 @@ module Karafka
       include Enumerable
       extend Forwardable
 
-      def_delegators :@accumulator, :[], :size, :empty?, :last, :<<, :map!, :sort_by!, :reverse!
+      def_delegators :@accumulator, :[], :size, :empty?, :last, :map!, :sort_by!, :reverse!
 
       # @param topics_array [Array<Karafka::Routing::Topic>] array with topics
       def initialize(topics_array)
         @accumulator = topics_array.dup
       end
 
+      # Adds a topic using copy-on-write: we publish a brand-new accumulator array instead of
+      # mutating the existing one in place.
+      #
+      # @param topic [Karafka::Routing::Topic] topic that should become part of this consumer
+      #   group's topics
+      # @return [self]
+      #
+      # @note Topics can be appended to this collection at runtime from one thread while other
+      #   threads iterate the same collection (for example during routing lookups). Swapping the
+      #   `@accumulator` reference atomically (rather than appending in place) guarantees a
+      #   concurrent `#each` always traverses a complete, immutable snapshot: it either sees the
+      #   newly added topic or it does not, but never a torn array. This holds on every Ruby
+      #   runtime, not only on MRI's GIL. The in-place mutators above (`map!`, `sort_by!`,
+      #   `reverse!`) only run while routes are built at boot, single-threaded, so they do not
+      #   need the same treatment.
+      def <<(topic)
+        @accumulator += [topic]
+        self
+      end
+
       # Yields each topic
       def each(&)
         @accumulator.each(&)

data/lib/karafka/server.rb

@@ -168,6 +168,15 @@ module Karafka
         # otherwise we would overwrite the shutdown process of the process that started Karafka
         return unless process.supervised?
 
+        # We intentionally do NOT flush or close the producer before the `Kernel.exit!` below.
+        # Forceful shutdown is a last resort - a job already exceeded `shutdown_timeout` - and it
+        # is expected that in-flight, async-buffered data (user `produce_async` calls and DLQ
+        # `produce_async` copies) may be lost. Flushing here would be risky precisely when it
+        # matters: the producer (or its connection pool) may itself be the resource that is blocked
+        # (e.g. an unreachable broker), so waiting on it could stall or hang the very forceful exit
+        # that must guarantee the process terminates. On a normal (graceful) shutdown the producer
+        # is still closed via the `ensure` block below, which `exit!` deliberately skips here.
+        #
         # exit! is not within the instrumentation as it would not trigger due to exit
         Kernel.exit!(forceful_exit_code)
       ensure