waterdrop 2.10.2 → 2.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1329d22d7f4f960b2df24949a040dd1e1b57ec73002ed779f3bcd0c5ad4dad20
-  data.tar.gz: b2b9868379d00dd3951df5a2e3b45bdce1b24295284fd5cb6cca410709a38a05
+  metadata.gz: 32f1c3b166f74db06dc75080800f4972207facc943d00d7eeab73a6c33668aed
+  data.tar.gz: 8de1c8eecf566e241bc375fea7de5f568cd055be35a83effcf5de23b7c92f468
 SHA512:
-  metadata.gz: 817244d3151a463cca84a1a7fefd298c5e355337325817430cdef5f5eb1c09013d20f801f9ef95b5e9182539f8cb550f25df62042bf93bc7826e9f50ee4b1caf
-  data.tar.gz: c60cc27e7b3787a9610229dacdcbe0440fa7332c5659f5e61348409b4ca2c507718a5bbd93bf5e088f8cc7a5d8b91da981eac0f377257ec0c4394e20dcd92994
+  metadata.gz: 901c6f6a576c38807eee47641675b12f9ef3907325fb2435c6b2e8656724c4f57be6593a10b7b85348b8cadd317216fe84ba182a9ccfaf3fa0208119c0609686
+  data.tar.gz: 25720c9a6ac31520400f181af47b389eae9def35f384cf9b1d89f8466c4cafc904c0390fbe14a1222c6917b3b26fbcc38d9580e9a0fa45db07d1cc9528c3bb85

data/.gitignore

@@ -19,6 +19,8 @@ db/*.sqlite3
 *.rbc
 *.sassc
 .redcar/
+.codex
+.claude
 .capistrano/
 .sass-cache
 /config/god/sidekiq.rb

data/.ruby-version

@@ -1 +1 @@
-4.0.5
+4.0.6

data/.yard-lint.yml

@@ -61,6 +61,34 @@ Documentation/BlankLineBeforeDefinition:
     SingleBlankLine: true
     OrphanedDocs: true
 
+Documentation/DuplicateNamespaceComment:
+  Description: Detects namespaces documented with a YARD comment in more than one file.
+  Enabled: true
+  Severity: error
+
+Documentation/UnderfilledLines:
+  Description: Detects documentation prose that wraps too early and wastes horizontal space.
+  Enabled: true
+  Severity: error
+  # Aligned with RuboCop's Layout/LineLength (Max: 100) so documentation prose
+  # wraps to the same width as code.
+  MaxLength: 100
+
+Documentation/LineLength:
+  Description: Detects documentation lines that exceed the maximum length.
+  Enabled: true
+  Severity: error
+  # Aligned with RuboCop's Layout/LineLength (Max: 100).
+  MaxLength: 100
+
+Documentation/TextSubstitution:
+  Description: Detects em/en-dashes in documentation and replaces them with hyphens.
+  Enabled: true
+  Severity: error
+  Substitutions:
+    "—": "-"    # em-dash (U+2014)
+    "–": "-"    # en-dash (U+2013)
+
 # Tags validators
 Tags/Order:
   Description: Enforces consistent ordering of YARD tags.
@@ -141,7 +169,7 @@ Tags/OptionTags:
 Tags/ExampleSyntax:
   Description: Validates Ruby syntax in @example tags.
   Enabled: true
-  Severity: warning
+  Severity: error
 
 Tags/RedundantParamDescription:
   Description: Detects meaningless parameter descriptions that add no value.

data/CHANGELOG.md

@@ -1,40 +1,49 @@
 # WaterDrop changelog
 
+## 2.10.4 (2026-08-26)
+- [Fix] Avoid a `FrozenError` on `Producer#close` when the producer is configured with a frozen string id (for example a frozen string literal, `config.id = "rspec"`).
+- [Fix] Emit `connection_pool.shutdown` only once when shutting down the global connection pool.
+- [Fix] Retry a produce that races a concurrent idempotent fatal-error reload instead of leaking a raw `Rdkafka::ClosedProducerError`/`Rdkafka::ClosedInnerError`.
+- [Maintenance] Stop the `#partition_count when topic does not exist` spec from flaking on slow CI runners by waiting for authoritative broker metadata before asserting the count.
+
+## 2.10.3 (2026-07-15)
+- [Feature] Add `wait_timeout_on_transaction_abort` (default `0`, disabled) - an opt-in mitigation for [librdkafka#4849](https://github.com/confluentinc/librdkafka/issues/4849) that waits (up to the given ms) for the first delivery to be acknowledged before aborting, avoiding a fatal `INVALID_TXN_STATE`. Off by default because it changes abort semantics: the awaited message is written to the log (aborted) instead of being purged.
+
 ## 2.10.2 (2026-06-15)
-- [Feature] Expose `Producer#current_variant` as a public method. It returns the variant active for the current dispatch on the current fiber - the custom variant while inside a `#with`/`#variant`-wrapped call, otherwise the producer's default variant - so middleware and instrumentation listeners running synchronously within a dispatch can read the effective per-dispatch settings (`topic_config`, `max_wait_timeout`, `default?`). The lookup is fiber-local and dispatch-scoped: outside a variant-wrapped call (or from an asynchronous delivery callback) it returns the default variant.
-- [Enhancement] Stop allocating one interpolated string per message in `LoggerListener` batch produce handlers. The quoted topic strings were only ever counted (quoting is a 1:1 mapping), never displayed, so counting the raw topic values yields the identical number with zero string allocations - relevant for large `produce_many_*` batches with the default logger listener attached.
+- [Feature] Expose `Producer#current_variant` as a public method returning the variant active for the current dispatch on the current fiber, so middleware and instrumentation listeners can read the effective per-dispatch settings (`topic_config`, `max_wait_timeout`, `default?`).
+- [Enhancement] Stop allocating one interpolated string per message in `LoggerListener` batch produce handlers.
 - [Enhancement] Use `Array#concat` in `Producer#buffer_many` instead of appending messages one by one.
-- [Enhancement] Skip building the `message.acknowledged` instrumentation payload in the delivery callback when nothing is subscribed to that event. The notifications bus already short-circuits on empty listeners, but only after the payload hash was allocated - once per delivered message on the polling thread. Mirrors the listener guard already used by the statistics callback. Late subscribers keep working as the check happens on each emission.
-- [Enhancement] Resolve the fiber-local variant once per `#produce` call and once per `#produce_many_sync` wait phase instead of re-resolving it for every usage and for every waited delivery handle. For a 1,000-message sync batch this removes ~2,000 redundant fiber-local lookups.
-- [Enhancement] Do not allocate the fiber-local variants hash on the `Producer#current_variant` read path. Previously every fiber that produced messages got a Hash pinned to it for the fiber's lifetime (per producer use), even when variants were never used - wasteful under fiber-per-request servers (Falcon, async). The hash is now only created by variant wrapper methods that actually need to write to it.
-- [Enhancement] Cache the variant validation contract in a constant instead of instantiating a new `Contracts::Variant` on every `Producer#with` / `Producer#variant` call (mirrors the existing `Transactions::CONTRACT` pattern).
-- [Enhancement] Cache the tombstone validation contract in a constant instead of instantiating a new `Contracts::Tombstone` per tombstone message, removing per-message allocations in the `tombstone_*` APIs (mirrors the existing `Transactions::CONTRACT` pattern).
-- [Enhancement] Replace explicit `Warning[:performance]` opt-in with a dynamic approach using `Warning.categories` (available since Ruby 3.4) to automatically enable all stable opt-in warning categories in the test suite, including `:strict_unused_block` introduced in Ruby 4.0.
-- [Fix] Prevent a deadlock between a transactional single-message dispatch and `#close`. A single `produce_sync`/`produce_async` on a transactional producer incremented the operations counter (which `#close` drains while holding `@transaction_mutex`) before acquiring `@transaction_mutex` for its per-message transaction - an inverted lock order. A dispatch that had counted itself but not yet taken `@transaction_mutex` could deadlock a concurrent `#close` permanently (the close wait loop has no timeout). Transactional dispatches now take `@transaction_mutex` before the operation is counted, matching `#close`'s lock order (`@transaction_mutex` -> `@operating_mutex` -> operations counter).
-- [Fix] Prevent a deadlock (`ThreadError: deadlock; recursive locking`) when closing an idempotent producer (with `reload_on_idempotent_fatal_error` enabled) that has buffered messages whose final flush surfaces a fatal librdkafka error. `#close` performs the final flush while already holding `@operating_mutex`, and the idempotent fatal-error reload tried to re-acquire that same mutex, leaving the producer stuck in `:closing` with the native client leaked. The idempotent reload is now skipped on the closing path, and the final buffer flush is best-effort so client teardown always completes.
-- [Fix] Make concurrent idempotent fatal-error reload thread-safe. When several threads shared an idempotent producer (with `reload_on_idempotent_fatal_error` enabled), a single fatal librdkafka condition failed all their in-flight produces at once and each entered the reload path; the second reload ran `reload!` after the first had already reset `@client` to `nil`, raising `NoMethodError`. The idempotent reload now bails out if another thread already reloaded (mirroring the transactional path's `return if @status.configured?` guard). Additionally, `Status#active?` now classifies the lifecycle from a single atomic read and `Producer#ensure_active!` branches on one snapshot, so a concurrent `configured -> connected` transition during a reload can no longer make `ensure_active!` raise `StatusInvalidError` for a valid, active producer.
-- [Fix] Stop `#flush_async` / `#flush_sync` from silently dropping valid buffered messages when the dispatch fails. `#flush` removes the batch from the internal buffer before dispatching it, and a failure (a single invalid message failing validation before anything is sent, or a mid-batch inline error such as queue full) previously discarded the entire taken batch - the removed messages were never restored. A failed flush now re-buffers the messages that never reached librdkafka (the whole batch on validation failure or on a transactional rollback, the unsent remainder otherwise) so they can be retried instead of being lost.
-- [Fix] Make `Producer#close` fork-safe so the GC finalizer inherited by a forked child can no longer close the parent's client. `#client` registers an `ObjectSpace` finalizer that calls `#close`; that finalizer is inherited across `fork`, and a child that inherited a used producer, never touched it, and exited normally would run `#close` in the child - flushing and closing (with the real rdkafka client, `rd_kafka_destroy` on a fork-inherited handle, i.e. undefined behavior) a client owned by the parent. `#close` now detects when it runs in a process other than the one that built the client, drops the inherited references and finalizer, and returns without touching the native client (matching the existing fork guard on the `#client` path).
-- [Fix] Guard the internal buffer appends in `Producer#buffer` and `Producer#buffer_many` with `@buffer_mutex`. The appends mutated the shared `@messages` buffer without the lock that `flush`/`purge`/`close` hold while swapping it for a fresh array, so a concurrent swap landing between reading `@messages` and appending could drop the message into an orphaned array that is never dispatched - silently losing buffered messages in the documented "buffer in one thread, flush in another" pattern.
-- [Fix] Stop a nested same-producer variant call from clobbering the outer variant inside a variant `transaction` block. `transaction` is the only variant-wrapped method that yields user code, so a variant call nested inside it (another `variant.produce_*`, or a raw producer dispatch in the same scope) used to delete the shared `Fiber.current.waterdrop_clients` entry on return, making the rest of the block silently fall back to the default variant and dispatch with default `topic_config` (timeouts, compression, partitioner) instead of the altered one. The wrapper now saves and restores the previous entry instead of unconditionally deleting it (still deleting when there was none, so the fiber-local hash does not accumulate stale keys).
-- [Fix] Stop `ConnectionPool#shutdown` and `#reload` from silently dropping in-flight messages. Both closed every pooled producer with `close!` (force), which flushes for the max wait timeout and then purges whatever has not drained - so on a slow or unreachable broker, queued `produce_async` messages were cancelled and lost with no delivery report. They now close producers gracefully by default (`#reload` always; `#shutdown` unless called with the new `force: true`), letting messages flush instead of being purged. Pass `pool.shutdown(force: true)` to keep the old force-and-purge behavior.
-- [Fix] Close a race in the FD poller where a producer registered while the last one was being torn down could be left permanently unpolled (sync produces hang until timeout, async deliveries are never acknowledged). The poller thread decided to exit (last producer unregistered) and cleared its thread reference in two separate, unsynchronized steps, so a `register` landing in that gap saw the still-alive exiting thread, skipped starting a fresh one, and then had its producer's state closed by the exiting thread's cleanup. The thread now decides to stop and clears its reference in a single mutex section, so a racing `register` either keeps it running or starts a fresh thread; and the exit cleanup runs only on an abnormal exit, since a normal exit always leaves an empty registry and so can never close a producer registered in the gap.
+- [Enhancement] Skip building the `message.acknowledged` instrumentation payload in the delivery callback when nothing is subscribed to that event.
+- [Enhancement] Resolve the fiber-local variant once per `#produce` call and once per `#produce_many_sync` wait phase instead of re-resolving it for every usage.
+- [Enhancement] Do not allocate the fiber-local variants hash on the `Producer#current_variant` read path when variants are never used.
+- [Enhancement] Cache the variant validation contract in a constant instead of instantiating a new `Contracts::Variant` on every `Producer#with` / `Producer#variant` call.
+- [Enhancement] Cache the tombstone validation contract in a constant instead of instantiating a new `Contracts::Tombstone` per tombstone message.
+- [Enhancement] Replace explicit `Warning[:performance]` opt-in with a dynamic approach using `Warning.categories` to enable all stable opt-in warning categories in the test suite.
+- [Fix] Prevent a deadlock between a transactional single-message dispatch and `#close` caused by an inverted lock order; transactional dispatches now take `@transaction_mutex` before the operation is counted.
+- [Fix] Prevent a deadlock (`ThreadError: deadlock; recursive locking`) when closing an idempotent producer (with `reload_on_idempotent_fatal_error` enabled) whose final buffer flush surfaces a fatal librdkafka error; the idempotent reload is now skipped on the closing path.
+- [Fix] Make concurrent idempotent fatal-error reload thread-safe so a second thread's reload can no longer raise `NoMethodError` after the first reset `@client`.
+- [Fix] Stop `#flush_async` / `#flush_sync` from silently dropping valid buffered messages when the dispatch fails; unsent messages are now re-buffered so they can be retried instead of being lost.
+- [Fix] Make `Producer#close` fork-safe so the GC finalizer inherited by a forked child can no longer flush and close the parent's client.
+- [Fix] Guard the internal buffer appends in `Producer#buffer` and `Producer#buffer_many` with `@buffer_mutex` so a concurrent flush/purge/close swap can no longer drop a message into an orphaned array.
+- [Fix] Stop a nested same-producer variant call from clobbering the outer variant inside a variant `transaction` block; the wrapper now saves and restores the previous fiber-local entry instead of deleting it.
+- [Fix] Stop `ConnectionPool#shutdown` and `#reload` from silently dropping in-flight messages. They now close producers gracefully by default (`#reload` always; `#shutdown` unless called with the new `force: true`); pass `pool.shutdown(force: true)` to keep the old force-and-purge behavior.
+- [Fix] Close a race in the FD poller where a producer registered while the last one was being torn down could be left permanently unpolled; the poller now decides to stop and clears its thread reference in a single mutex section.
 
 ## 2.10.1 (2026-05-25)
 - [Fix] Prevent `Producer#close` from raising `ThreadError: can't be called from trap context` when invoked from a Ruby signal trap context (e.g. Puma's `after_stopped` DSL hook in single mode). `close` now detects this case and delegates to a background thread, joining it so the caller blocks until the producer is fully closed (#866).
 
 ## 2.10.0 (2026-05-07)
-- [Fix] Clean up native rdkafka client, global instrumentation callbacks, and poller registration when `init_transactions` fails during producer client construction. Previously, each failed attempt permanently leaked native threads, pipe file descriptors, and callback registry entries because the started `rd_kafka_t` handle was abandoned without being destroyed.
-- **[Breaking]** Skip emitting librdkafka statistics when nothing is subscribed to `statistics.emitted` at the time the underlying rdkafka client is constructed. When no listener is present at build time, `statistics.interval.ms` is forced to `0` regardless of user configuration and the statistics callback is not registered, saving substantial allocations in the hot path (no JSON parsing, no statistics hash materialization, no decoration work). To use statistics, subscribe a listener to `statistics.emitted` BEFORE the first producer use (before the underlying client is lazily initialized).
-- **[Breaking]** Raise `WaterDrop::Errors::StatisticsNotEnabledError` when attempting to subscribe to `statistics.emitted` (either via block or via a listener that responds to `on_statistics_emitted`) on a monitor where librdkafka statistics have been disabled at client build time. This replaces the "silent nothing" failure mode with an immediate, actionable error that pinpoints the timing mistake.
+- [Fix] Clean up native rdkafka client, global instrumentation callbacks, and poller registration when `init_transactions` fails during producer client construction, so failed attempts no longer leak native threads, file descriptors, and callback registry entries.
+- **[Breaking]** Skip emitting librdkafka statistics when nothing is subscribed to `statistics.emitted` at the time the underlying rdkafka client is constructed (`statistics.interval.ms` is forced to `0`), saving substantial allocations in the hot path. To use statistics, subscribe a listener to `statistics.emitted` BEFORE the first producer use.
+- **[Breaking]** Raise `WaterDrop::Errors::StatisticsNotEnabledError` when attempting to subscribe to `statistics.emitted` on a monitor where librdkafka statistics have been disabled at client build time, replacing the previous "silent nothing" failure mode.
 - [Feature] Add tombstone API (`#tombstone_sync`, `#tombstone_async`, `#tombstone_many_sync`, `#tombstone_many_async`) for producing tombstone records (nil-payload messages) with required key and partition validation. Works with variants.
-- [Fix] Add `ensure_same_process!` to `Poller#unregister` for fork safety. Without this, a child process that inherited a pre-fork producer would deadlock on `producer.close` because `unregister` waited on a latch the dead parent poller thread would never release.
+- [Fix] Add `ensure_same_process!` to `Poller#unregister` for fork safety, so a child that inherited a pre-fork producer no longer deadlocks on `producer.close`.
 
 ## 2.9.0 (2026-04-08)
 - [Fix] Use `delete` in the variant ensure block to avoid leaving stale nil entries in `Fiber.current.waterdrop_clients` and prevent memory leaks in long-running processes (#836).
 - [Fix] Exclude test files, `.github/`, and `log/` directories from gem releases to reduce package size.
 - **[Breaking]** Switch default polling mode from `:thread` to `:fd`. If you experience any issues, you can revert to the previous behavior by setting `config.polling.mode = :thread`. The `:thread` mode will be deprecated in 2.10 and removed in 2.11.
-- **[Breaking]** Statistics decorator now only decorates keys used by the built-in Datadog metrics listener (`tx`, `txretries`, `txerrs`, `rxerrs`) and skips unused subtrees (`topics`, broker window stats). This reduces decoration cost by ~425x on large clusters (10 brokers, 20 topics, 2000 partitions). Users who rely on other `_d` or `_fd` keys in custom instrumentation should provide a custom decorator via `config.statistics_decorator`.
+- **[Breaking]** Statistics decorator now only decorates keys used by the built-in Datadog metrics listener (`tx`, `txretries`, `txerrs`, `rxerrs`) and skips unused subtrees, greatly reducing decoration cost on large clusters. Users who rely on other `_d` or `_fd` keys in custom instrumentation should provide a custom decorator via `config.statistics_decorator`.
 - [Feature] Add `config.statistics_decorator` setting to allow full control over the `StatisticsDecorator` instance used for statistics decoration. Users can provide a custom decorator with different `only_keys` and `excluded_keys` to match their instrumentation needs.
 - [Change] Upscale default timeout values 3x closer to librdkafka defaults to prevent intermediate timeouts during node recovery (`message.timeout.ms`: 50s → 150s, `transaction.timeout.ms`: 55s → 165s, `max_wait_timeout`: 60s → 180s).
 

data/Gemfile.lint.lock

@@ -58,11 +58,11 @@ GEM
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.2.0)
-    yard (0.9.38)
-    yard-lint (1.4.0)
+    yard (0.9.45)
+    yard-lint (1.10.1)
       yard (~> 0.9)
       zeitwerk (~> 2.6)
-    zeitwerk (2.7.4)
+    zeitwerk (2.8.2)
 
 PLATFORMS
   ruby
@@ -79,6 +79,7 @@ DEPENDENCIES
 
 CHECKSUMS
   ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
+  bundler (4.0.17) sha256=214e21431b5665dd2f99df8a5511c6b151d7a72e8015c8b38f8b775b61cbb6c1
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
@@ -100,9 +101,9 @@ CHECKSUMS
   standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
   unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
-  yard (0.9.38) sha256=721fb82afb10532aa49860655f6cc2eaa7130889df291b052e1e6b268283010f
-  yard-lint (1.4.0) sha256=7dd88fbb08fd77cb840bea899d58812817b36d92291b5693dd0eeb3af9f91f0f
-  zeitwerk (2.7.4) sha256=2bef90f356bdafe9a6c2bd32bcd804f83a4f9b8bc27f3600fff051eb3edcec8b
+  yard (0.9.45) sha256=52e211493f7cb8a3ebf7e104a25a1e73937a3103092545d34cb88fafebb3dc51
+  yard-lint (1.10.1) sha256=172ea1905304580bd515c093d82831ec4889e725100c49d5bf77e62101f49129
+  zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12
 
 BUNDLED WITH
-   2.6.9
+  4.0.17

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.10.2)
+    waterdrop (2.10.4)
       karafka-core (>= 2.5.12, < 3.0.0)
       karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
@@ -12,15 +12,14 @@ GEM
     byebug (13.0.0)
       reline (>= 0.6.0)
     connection_pool (3.0.2)
-    docile (1.4.1)
     drb (2.2.3)
     ffi (1.17.4)
     io-console (0.8.2)
-    json (2.19.7)
-    karafka-core (2.5.13)
+    json (2.21.2)
+    karafka-core (2.6.2)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.27.2)
+    karafka-rdkafka (0.28.0)
       ffi (~> 1.17.1)
       json (> 2.0)
       logger
@@ -39,14 +38,9 @@ GEM
     reline (0.6.3)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.13.2)
-    simplecov_json_formatter (0.1.4)
+    simplecov (1.1.1)
     warning (1.6.0)
-    zeitwerk (2.8.2)
+    zeitwerk (2.8.3)
 
 PLATFORMS
   ruby
@@ -65,13 +59,12 @@ DEPENDENCIES
 CHECKSUMS
   byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
-  docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
   ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
-  json (2.19.7) sha256=fe432c8639f6efff69f9d73b518a3705d9581ab93156f981ea72806e1e5bcc3e
-  karafka-core (2.5.13) sha256=0acec083043bb6166c4b647a7458091cc7b08066d3b92a026932925ec7e07f61
-  karafka-rdkafka (0.27.2) sha256=3ccce96306642be70bff8168e4e737fc10f2ffae20bc0ff0a43d88dbb7452d31
+  json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a
+  karafka-core (2.6.2) sha256=c2fd7f277201b8ca97b824b364ad76bf776b8f5527bc422dafb71f7ae48d3a13
+  karafka-rdkafka (0.28.0) sha256=fd845140c42341c5892215cac1e8adacda53b0461688f2e16379d82cb33eb006
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
   minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1
@@ -81,12 +74,10 @@ CHECKSUMS
   rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
   reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
   ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
-  simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
-  simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
-  simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
+  simplecov (1.1.1) sha256=25825ef13f0b2e74694d769817dad6ab8e90131dabdaa666e522fea105521e78
   warning (1.6.0) sha256=a49cdfae19fb77d19afff2efbe45f8ab759e9cd25b4e4ce2c79dbaf46bdb6c9e
-  waterdrop (2.10.2)
-  zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12
+  waterdrop (2.10.4)
+  zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5
 
 BUNDLED WITH
   4.0.6

data/bin/verify_kafka_warnings

@@ -7,6 +7,18 @@ allowed_patterns=(
    "Performing controller activation"
    "registered with feature metadata.version"
    "TOPIC_ALREADY_EXISTS"
+   # The transactional_abort_after_commit_race and transactional_abort_race_canary integration specs
+   # deliberately reproduce librdkafka#4849: they abort with the first produce still in flight, so
+   # the abort's EndTxn can reach the coordinator before AddPartitionsToTxn registers the new
+   # transaction. The coordinator then still holds the *previous* transaction's COMPLETE_COMMIT state
+   # and rejects the ABORT marker, logging this warning. It is the broker-side signature of the very
+   # defect those specs exist to reproduce, so it is expected there - and scoped to their
+   # transactional ids so we still catch the same warning anywhere else.
+   #
+   # One entry per spec rather than an alternation: `\|` is a GNU extension to POSIX BRE, so on BSD
+   # grep (macOS) it would match nothing and the allowance would silently do nothing at all.
+   "tx-abort-race-id.*received transaction marker result to send: ABORT"
+   "tx-abort-canary-id.*received transaction marker result to send: ABORT"
 )
 
 # Get all warnings

data/docker-compose.oauth.yml

@@ -18,7 +18,7 @@ services:
       start_period: 90s
 
   kafka-oauth:
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
     container_name: kafka-oauth
     depends_on:
       keycloak:

data/docker-compose.sasl.yml

@@ -1,6 +1,6 @@
 services:
   kafka-sasl:
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
     container_name: kafka-sasl
     ports:
       - "9095:9095"

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
 
     ports:
       - 9092:9092

data/lib/waterdrop/clients/buffered.rb

@@ -80,8 +80,7 @@ module WaterDrop
         @topics[topic]
       end
 
-      # Clears internal buffer
-      # Used in between specs so messages do not leak out
+      # Clears internal buffer. Used in between specs so messages do not leak out
       def reset
         @transaction_level = 0
         @transaction_active = false

data/lib/waterdrop/clients/rdkafka.rb

@@ -51,7 +51,7 @@ module WaterDrop
         # saves a significant number of allocations on the Ruby side (no JSON parsing, no
         # statistics hash materialization, no decorator work). Any listener subscribed after
         # the client has been built will not receive `statistics.emitted` events because
-        # librdkafka never emits them in the first place — to use statistics, subscribe a
+        # librdkafka never emits them in the first place - to use statistics, subscribe a
         # listener BEFORE the first producer use.
         #
         # When statistics end up disabled (either because the user explicitly set the interval
@@ -94,7 +94,7 @@ module WaterDrop
 
         # Registers the global callbacks (statistics, error, oauth refresh) for this producer
         # on the shared `Karafka::Core::Instrumentation` managers. The statistics callback is
-        # only registered when librdkafka is actually going to emit statistics — otherwise it
+        # only registered when librdkafka is actually going to emit statistics - otherwise it
         # would never fire and would only waste memory and a manager slot.
         #
         # @param producer [WaterDrop::Producer]

data/lib/waterdrop/config.rb

@@ -36,7 +36,7 @@ module WaterDrop
       constructor: ->(id) { id || "waterdrop-#{SecureRandom.hex(6)}" }
     )
     # option [Instance] logger that we want to use
-    # @note Due to how rdkafka works, this setting is global for all the producers
+    # Due to how rdkafka works, this setting is global for all the producers
     setting(
       :logger,
       default: false,
@@ -95,6 +95,35 @@ module WaterDrop
     # option [Integer] How many times to attempt reloading on transactional fatal error before
     #   giving up. This prevents infinite reload loops if the producer never recovers.
     setting :max_attempts_on_transaction_fatal_error, default: 10
+    # option [Numeric] How long to wait **at most** (in ms) for the first delivery of a transaction
+    #   to be acknowledged before we abort that transaction. `0` (the default) disables the wait.
+    #
+    #   This is a ceiling, not a fixed delay: the wait ends as soon as the delivery is acknowledged,
+    #   which under normal conditions is immediate. The full timeout is only ever spent when the
+    #   delivery never arrives at all.
+    #
+    #   This is an opt-in mitigation for a librdkafka defect. librdkafka only flags a transaction as
+    #   ongoing at the coordinator once the `AddPartitionsToTxn` **response** arrives, but it sends
+    #   `EndTxn` as soon as that request has merely been **sent**. Aborting with the first produce
+    #   still in flight can therefore hit a coordinator that does not consider the transaction
+    #   started yet, failing the abort with a fatal `INVALID_TXN_STATE`.
+    #   See https://github.com/confluentinc/librdkafka/issues/4849
+    #
+    #   A single acknowledged delivery proves its partition completed registration, and that alone
+    #   makes the coordinator accept `EndTxn` - no matter how many partitions the transaction spans.
+    #   So waiting for one delivery before aborting closes the race.
+    #
+    #   It is off by default because it changes abort semantics: waiting for that ack means the
+    #   first message is actually **delivered** (aborted, so invisible to `read_committed`
+    #   consumers) instead of being **purged**. Its delivery handle then reports a real offset
+    #   rather than a `Purged in queue` error, and no `message.purged` event is emitted for it,
+    #   while the remaining messages of the same transaction are still purged. Enable this only if
+    #   you hit the defect and prefer that trade over the fatal (which stays recoverable through
+    #   `reload_on_transaction_fatal_error` either way).
+    #
+    #   The wait is bounded and best-effort: if it expires (broker down, message timeout) we abort
+    #   exactly as if it were disabled.
+    setting :wait_timeout_on_transaction_abort, default: 0
     # option [Array<Symbol>] List of fatal error codes that should NOT trigger producer reload.
     #   These errors represent states that cannot be recovered by simply recreating the client.
     #
@@ -140,7 +169,7 @@ module WaterDrop
     # option [Class] class for usage when creating the underlying client used to dispatch messages
     setting :client_class, default: Clients::Rdkafka
     # rdkafka options
-    # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    # See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}
     # Middleware chain that can be expanded with useful middleware steps
     setting(

data/lib/waterdrop/connection_pool.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# WaterDrop library
 module WaterDrop
   # Connection pool wrapper for WaterDrop producers using the proven connection_pool gem.
   #
@@ -119,15 +118,11 @@ module WaterDrop
       def shutdown(force: false)
         return unless @default_pool
 
-        pool = @default_pool
+        # The instrumentation event is emitted by the instance-level #shutdown we delegate to
+        # (with `pool: self`, i.e. this same global pool). We deliberately do not emit it again
+        # here - doing so fired `connection_pool.shutdown` twice for a single global shutdown.
         @default_pool.shutdown(force: force)
         @default_pool = nil
-
-        # Emit global event for pool shutdown
-        WaterDrop.instrumentation.instrument(
-          "connection_pool.shutdown",
-          pool: pool
-        )
       end
 
       # Alias for shutdown to align with producer API

data/lib/waterdrop/contracts/config.rb

@@ -30,6 +30,7 @@ module WaterDrop
       required(:max_attempts_on_idempotent_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
       required(:wait_backoff_on_transaction_fatal_error) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:max_attempts_on_transaction_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:wait_timeout_on_transaction_abort) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:non_reloadable_errors) do |val|
         val.is_a?(Array) && val.all?(Symbol)
       end

data/lib/waterdrop/instrumentation/monitor.rb

@@ -2,9 +2,8 @@
 
 module WaterDrop
   module Instrumentation
-    # WaterDrop instrumentation monitor that we use to publish events
-    # By default uses our internal notifications bus but can be used with
-    # `ActiveSupport::Notifications` as well
+    # WaterDrop instrumentation monitor that we use to publish events. By default uses our internal
+    # notifications bus but can be used with `ActiveSupport::Notifications` as well
     class Monitor < ::Karafka::Core::Monitoring::Monitor
       # Event name for librdkafka statistics emissions
       STATISTICS_EVENT = "statistics.emitted"
@@ -29,8 +28,8 @@ module WaterDrop
       # Marks this monitor as no longer accepting new subscriptions to `statistics.emitted`.
       # Called by the rdkafka client builder when it decides to leave librdkafka statistics
       # disabled (because no listener was present at build time). Any subsequent attempt to
-      # subscribe to `statistics.emitted` — either via a block or via a listener object that
-      # responds to `on_statistics_emitted` — will raise
+      # subscribe to `statistics.emitted` - either via a block or via a listener object that
+      # responds to `on_statistics_emitted` - will raise
       # `WaterDrop::Errors::StatisticsNotEnabledError` instead of silently doing nothing.
       def freeze_statistics_listeners!
         @statistics_listeners_frozen = true

data/lib/waterdrop/polling/config.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-# WaterDrop main module
 module WaterDrop
-  # Namespace for FD-based polling components
   module Polling
     # Configuration for the global FD poller singleton
     # These settings apply to all producers using FD polling mode

data/lib/waterdrop/polling/poller.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 module WaterDrop
-  # Namespace for FD-based polling components
-  # Contains the global Poller singleton and State class for managing producer polling
   module Polling
     # Global poller singleton that manages a single polling thread for all FD-mode producers
     # This replaces librdkafka's native background polling threads with a single Ruby thread
@@ -272,7 +270,8 @@ module WaterDrop
 
       # Collects all IOs to monitor and builds a mapping from IO to State
       # Uses cached arrays when possible to avoid allocations in the hot path
-      # @return [Array<Array<IO>, Hash{IO => State}, Array<State>>] tuple of ios, io-to-state map, states
+      # @return [Array<Array<IO>, Hash{IO => State}, Array<State>>] tuple of ios, io-to-state map,
+      #   states
       def collect_readable_ios
         # Fast path: return cached result if not dirty (no mutex needed)
         # Safe because @cached_result is frozen and assigned atomically
@@ -345,9 +344,8 @@ module WaterDrop
         end
       end
 
-      # Polls all registered producers
-      # Called when IO.select times out to ensure periodic polling happens
-      # This ensures OAuth token refresh and statistics callbacks fire for all producers
+      # Polls all registered producers. Called when IO.select times out to ensure periodic polling
+      # happens. This ensures OAuth token refresh and statistics callbacks fire for all producers
       def poll_all_producers
         @cached_states.each { |state| poll_producer(state) }
       end

data/lib/waterdrop/polling/state.rb

@@ -26,7 +26,8 @@ module WaterDrop
       # @param client [Rdkafka::Producer] the rdkafka producer client
       # @param monitor [Object] the producer's monitor for error reporting
       # @param max_poll_time [Integer] max time in ms to poll per cycle
-      # @param periodic_poll_interval [Integer] max time in ms before this producer needs periodic poll
+      # @param periodic_poll_interval [Integer] max time in ms before this producer needs periodic
+      #   poll
       # @raise [StandardError] if queue pipe setup fails (FD mode requires this to work)
       def initialize(producer_id, client, monitor, max_poll_time, periodic_poll_interval)
         @producer_id = producer_id
@@ -106,9 +107,8 @@ module WaterDrop
         @last_stale_result = (now - @last_poll_time) >= @periodic_poll_interval
       end
 
-      # Signals the poller to remove this producer
-      # Called from any thread when the producer is being closed
-      # Sets closing flag BEFORE signaling to ensure poller sees it
+      # Signals the poller to remove this producer. Called from any thread when the producer is
+      # being closed. Sets closing flag BEFORE signaling to ensure poller sees it
       def signal_close
         @closing = true
         @queue_pipe.signal
@@ -133,9 +133,8 @@ module WaterDrop
         @close_latch.release!
       end
 
-      # Waits for this state to be closed
-      # Used by unregister to ensure synchronous cleanup before returning
-      # This matches the threaded polling behavior which drains without timeout
+      # Waits for this state to be closed. Used by unregister to ensure synchronous cleanup before
+      # returning. This matches the threaded polling behavior which drains without timeout
       def wait_for_close
         @close_latch.wait
       end

data/lib/waterdrop/polling.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# WaterDrop main module
 module WaterDrop
   # Namespace for FD-based polling components
   # Contains the global Poller singleton and State class for managing producer polling

data/lib/waterdrop/producer/tombstone.rb

@@ -16,7 +16,7 @@ module WaterDrop
       # Produces a tombstone message to Kafka and waits for it to be delivered
       #
       # @param message [Hash] hash with at least `:topic`, `:key`, and `:partition` keys.
-      #   `:payload` is not accepted — it will be silently removed if present.
+      #   `:payload` is not accepted - it will be silently removed if present.
       #
       # @return [Rdkafka::Producer::DeliveryReport] delivery report
       #
@@ -28,7 +28,7 @@ module WaterDrop
       # Produces a tombstone message to Kafka and does not wait for results
       #
       # @param message [Hash] hash with at least `:topic`, `:key`, and `:partition` keys.
-      #   `:payload` is not accepted — it will be silently removed if present.
+      #   `:payload` is not accepted - it will be silently removed if present.
       #
       # @return [Rdkafka::Producer::DeliveryHandle] delivery handle
       #

data/lib/waterdrop/producer/transactions.rb

@@ -106,9 +106,11 @@ module WaterDrop
             # This is why we catch this here
             begin
               with_transactional_error_handling(:abort) do
-                transactional_instrument(:aborted) do
-                  client.abort_transaction
-                end
+                # Outside of the `aborted` instrumentation on purpose: that event is supposed to
+                # measure `client.abort_transaction`, not our wait
+                transactional_await_first_delivery(e)
+
+                transactional_instrument(:aborted) { client.abort_transaction }
               end
             rescue => e
               # If something from rdkafka leaks here, it means there was a non-retryable error that
@@ -121,6 +123,13 @@ module WaterDrop
             transactional_reload_client_if_needed(e)
 
             raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
+          ensure
+            # The first delivery handle is scoped to the transaction that produced it and is only
+            # ever read by the abort path (which runs in the rescue above, before this). Clearing it
+            # here - on commit, on abort and on a re-raise alike - keeps it from outliving its
+            # transaction: otherwise a committed transaction would leave its handle (and everything
+            # the delivery report references) pinned to the producer until the next one begins.
+            @transaction_first_handle = nil
           end
         end
       end
@@ -255,6 +264,12 @@ module WaterDrop
         if e.abortable? && allow_abortable
           # Always attempt to abort but if aborting fails with an abortable error, do not attempt
           # to abort from abort as this could create an infinite loop
+          #
+          # We deliberately do NOT wait for the first delivery here (unlike the abort in
+          # `#transaction`): we only get here because librdkafka reported an abortable error, so the
+          # transaction is already in an error state where it no longer delivers. The queued messages
+          # are purged by the abort itself, so the handle would never resolve and we would just stall
+          # for the whole `wait_timeout_on_transaction_abort` on a path that is already recovering.
           with_transactional_error_handling(:abort, allow_abortable: false) do
             transactional_instrument(:aborted) { client.abort_transaction }
           end
@@ -263,6 +278,61 @@ module WaterDrop
         raise
       end
 
+      # Waits (bounded) for the first delivery of the current transaction before we abort it.
+      #
+      # librdkafka only marks a transaction as ongoing at the coordinator once the
+      # `AddPartitionsToTxn` **response** comes back, but it fires `EndTxn` as soon as that request
+      # has merely been **sent** (it gates on `txn_req_cnt`, bumped on send). Aborting with the
+      # first produce still in flight can therefore reach a coordinator that does not yet consider
+      # the transaction started, which fails the abort with a fatal `INVALID_TXN_STATE`. See
+      # https://github.com/confluentinc/librdkafka/issues/4849
+      #
+      # A delivered message proves its partition completed registration, and that alone puts the
+      # transaction in an `ongoing` state at the coordinator - so a single acknowledged delivery is
+      # enough to make `EndTxn` valid, regardless of how many partitions the transaction spans.
+      #
+      # This is best-effort and never fatal: if the delivery does not materialize within the timeout
+      # (broker down, message timeout, purge) we abort exactly as before rather than hanging. The
+      # fatal, should it still happen, remains recoverable through the client reload.
+      #
+      # @param error [Exception] the error that is causing us to abort this transaction
+      def transactional_await_first_delivery(error)
+        handle = @transaction_first_handle
+        timeout = config.wait_timeout_on_transaction_abort
+
+        return if handle.nil?
+        return if timeout.zero?
+        # Only worth waiting when the transaction is still healthy, that is when we abort because the
+        # user asked us to (`AbortTransaction`) or because their own code raised. Once librdkafka
+        # itself reported an error, the transaction no longer delivers: the queued messages are
+        # purged by the abort, so this handle would never resolve and we would stall for the full
+        # timeout on a path that is already recovering from an error.
+        return if transactional_error?(error)
+
+        # An already delivered handle (the common case - any sync dispatch, or a transaction that did
+        # some work before aborting) short-circuits inside `#wait` itself, so there is nothing to
+        # guard against here: it returns immediately without blocking.
+        wait(handle, max_wait_timeout: timeout, raise_response_error: false)
+      rescue ::Rdkafka::AbstractHandle::WaitTimeoutError, ::Rdkafka::RdkafkaError
+        # These two mean the same thing for us: we could not confirm the registration, either because
+        # the delivery did not arrive in time or because it failed outright. Aborting is still the
+        # right move, we just lose the mitigation for this one transaction.
+        #
+        # Deliberately narrow. A broader rescue would also swallow bugs of our own making (a
+        # `NoMethodError` on a handle we mis-tracked, say) and turn them into a silently skipped
+        # mitigation that nobody would ever notice.
+        nil
+      end
+
+      # @param error [Exception] error that caused the abort
+      # @return [Boolean] did this error come from librdkafka, meaning the transaction is already in
+      #   an error state and will not deliver anything anymore
+      def transactional_error?(error)
+        return true if error.is_a?(::Rdkafka::RdkafkaError)
+
+        error.cause.is_a?(::Rdkafka::RdkafkaError)
+      end
+
       # Reloads the underlying client instance if needed and allowed
       #
       # This should be used only in transactions as only then we can get fatal transactional

data/lib/waterdrop/producer/variant.rb

@@ -122,7 +122,7 @@ module WaterDrop
           topic_config: topic_config,
           # We pass this to validation, to make sure no-one alters the `acks` value when operating
           # in the transactional mode as it causes librdkafka to crash ruby
-          # @see https://github.com/confluentinc/librdkafka/issues/4710
+          # See https://github.com/confluentinc/librdkafka/issues/4710
           transactional: @producer.transactional?,
           # We pass this for a similar reason as above
           idempotent: @producer.idempotent?

data/lib/waterdrop/producer.rb

@@ -67,6 +67,7 @@ module WaterDrop
       @poller = nil
       @idempotent_fatal_error_attempts = 0
       @transaction_fatal_error_attempts = 0
+      @transaction_first_handle = nil
 
       @status = Status.new
       @messages = []
@@ -92,7 +93,9 @@ module WaterDrop
         .setup(...)
         .config
 
-      @id = @config.id
+      # dup so a frozen id (e.g. a frozen string literal) isn't used as the finalizer target, since
+      # ObjectSpace.(un)define_finalizer mutates the object and would raise FrozenError on #close
+      @id = @config.id.to_s.dup
       @monitor = @config.monitor
       @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
       @default_variant = Variant.new(self, default: true)
@@ -589,12 +592,13 @@ module WaterDrop
     #
     # For a transactional producer we wrap the whole dispatch (including the operations-counter
     # bookkeeping) in `transaction`, so `@transaction_mutex` is acquired BEFORE
-    # `@operations_in_progress` is incremented. This makes `#produce` acquire locks in the same order
-    # as `#close` (`@transaction_mutex` -> `@operating_mutex` -> operations counter) and removes a
-    # lock-order inversion: without it, a dispatch that had already counted itself could block forever
-    # on `@transaction_mutex` held by a concurrent `#close` that was itself waiting for the operations
-    # counter to drain. When we already own the transaction lock (inside an explicit transaction block
-    # or the closing flush) the order is already correct, so we dispatch directly.
+    # `@operations_in_progress` is incremented. This makes `#produce` acquire locks in the same
+    # order as `#close` (`@transaction_mutex` -> `@operating_mutex` -> operations counter) and
+    # removes a lock-order inversion: without it, a dispatch that had already counted itself could
+    # block forever on `@transaction_mutex` held by a concurrent `#close` that was itself waiting
+    # for the operations counter to drain. When we already own the transaction lock (inside an
+    # explicit transaction block or the closing flush) the order is already correct, so we dispatch
+    # directly.
     #
     # @param message [Hash] message we want to send
     # @param label [String] short name of the public dispatch method (e.g. `"produce_sync"`) that
@@ -647,10 +651,39 @@ module WaterDrop
         client.produce(**message)
       end
 
+      # Remember the first delivery handle of the current transaction. Aborting while the very first
+      # produce is still in flight is what triggers librdkafka#4849, so the abort path waits on this
+      # handle to confirm the transaction is registered at the coordinator. See
+      # `#transactional_await_first_delivery`.
+      @transaction_first_handle ||= result if transactional? && @transaction_mutex.owned?
+
       # Reset attempts counter on successful produce
       @idempotent_fatal_error_attempts = 0
 
       result
+    rescue Rdkafka::ClosedProducerError, Rdkafka::ClosedInnerError
+      # A concurrent idempotent fatal-error reload closed the underlying client while this produce
+      # was already in flight. Unlike `#close`/`#disconnect`, which drain `@operations_in_progress`
+      # before closing the client, the idempotent reload swaps `@client` out from under sibling
+      # threads that have already passed the `@operating_mutex` gate and are inside `client.produce`.
+      # Racing with `@client.close`, such a thread sees either a producer already flagged closed
+      # (`ClosedProducerError`) or a nil inner librdkafka handle (`ClosedInnerError`) - which one
+      # depends purely on how far `close` has progressed.
+      #
+      # This is a benign, recoverable transient rather than a produce failure: the client has just
+      # been (or is being) rebuilt, so we retry the dispatch against the fresh client instead of
+      # surfacing a raw "closed producer" error to the caller - which would defeat the whole point
+      # of the transparent reload. Only the idempotent reload path closes the client with produces
+      # in flight, so we scope the retry to that configuration; anywhere else a closed client is a
+      # genuine error and must propagate. The retried pass re-runs `ensure_active!`, which raises
+      # `ProducerClosedError` once the producer is genuinely closing/closed, so this cannot spin
+      # forever.
+      raise unless config.reload_on_idempotent_fatal_error
+      raise if transactional?
+
+      @operations_in_progress.decrement
+
+      retry
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Check if this is a fatal error on an idempotent producer and we should reload.
       #

data/lib/waterdrop/version.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
-# WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = "2.10.2"
+  VERSION = "2.10.4"
 end

data/lib/waterdrop.rb

@@ -10,7 +10,6 @@ require "singleton"
 require "karafka-core"
 require "pathname"
 
-# WaterDrop library
 module WaterDrop
   class << self
     # @return [String] root path of this gem

data/package-lock.json

@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz",
-      "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==",
+      "version": "1.7.1",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
+      "integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.10.2
+  version: 2.10.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -160,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Kafka messaging made easy!
 test_files: []