waterdrop 2.10.2 → 2.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1329d22d7f4f960b2df24949a040dd1e1b57ec73002ed779f3bcd0c5ad4dad20
-  data.tar.gz: b2b9868379d00dd3951df5a2e3b45bdce1b24295284fd5cb6cca410709a38a05
+  metadata.gz: ec39a827b8ea9216ef39589a8ce2c8261ce29b5ba1386ddb513c2ca393b5ade1
+  data.tar.gz: d37a7617f72d101f75c5d02f4c77627757ad5254322ecbf8bf1c30b0b502fb98
 SHA512:
-  metadata.gz: 817244d3151a463cca84a1a7fefd298c5e355337325817430cdef5f5eb1c09013d20f801f9ef95b5e9182539f8cb550f25df62042bf93bc7826e9f50ee4b1caf
-  data.tar.gz: c60cc27e7b3787a9610229dacdcbe0440fa7332c5659f5e61348409b4ca2c507718a5bbd93bf5e088f8cc7a5d8b91da981eac0f377257ec0c4394e20dcd92994
+  metadata.gz: e65469f5af00b6976f8d6bcc7efecbb8bc7ce3e84418928d6364197c2b84c600730bb36cfa0ffd2333ad7a726367406aea69fa56d9ecc796b1a31a8b5cb11f94
+  data.tar.gz: 654c121690450afdbb356527cbfb142bacc0b7eddbd3dd0c69017d5564cbdeb0712647cd8b13c27180a7cc94a23cca4049d748935e0ca6ccef068f5f55c079d7

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # WaterDrop changelog
 
+## 2.10.3 (2026-07-15)
+- [Feature] Add `wait_timeout_on_transaction_abort` (default `0`, disabled) - an opt-in mitigation for [librdkafka#4849](https://github.com/confluentinc/librdkafka/issues/4849). librdkafka only marks a transaction as ongoing at the coordinator once the `AddPartitionsToTxn` **response** arrives, but it fires `EndTxn` as soon as that request has merely been **sent**. Aborting while the first produce is still in flight can therefore reach a coordinator that does not yet consider the transaction started, failing the abort with a fatal `INVALID_TXN_STATE` that poisons the client and forces a full reload. When set to a positive value, we wait for the first delivery of the transaction to be acknowledged before aborting, for at most that long (in ms) - it is a ceiling rather than a fixed delay, so the wait ends as soon as the delivery is acknowledged. A single ack proves its partition completed registration, which is enough for the coordinator to accept `EndTxn` regardless of how many partitions the transaction spans. It is off by default because it changes abort semantics: the awaited message is really written to the log (aborted, hence invisible to `read_committed` consumers) instead of being purged, so its delivery handle reports a real offset rather than a `Purged in queue` error and no `message.purged` is emitted for it. The wait is bounded and best-effort - if it expires we abort exactly as before, and the fatal remains recoverable through `reload_on_transaction_fatal_error`.
+
 ## 2.10.2 (2026-06-15)
 - [Feature] Expose `Producer#current_variant` as a public method. It returns the variant active for the current dispatch on the current fiber - the custom variant while inside a `#with`/`#variant`-wrapped call, otherwise the producer's default variant - so middleware and instrumentation listeners running synchronously within a dispatch can read the effective per-dispatch settings (`topic_config`, `max_wait_timeout`, `default?`). The lookup is fiber-local and dispatch-scoped: outside a variant-wrapped call (or from an asynchronous delivery callback) it returns the default variant.
 - [Enhancement] Stop allocating one interpolated string per message in `LoggerListener` batch produce handlers. The quoted topic strings were only ever counted (quoting is a 1:1 mapping), never displayed, so counting the raw topic values yields the identical number with zero string allocations - relevant for large `produce_many_*` batches with the default logger listener attached.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.10.2)
+    waterdrop (2.10.3)
       karafka-core (>= 2.5.12, < 3.0.0)
       karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
@@ -12,12 +12,11 @@ GEM
     byebug (13.0.0)
       reline (>= 0.6.0)
     connection_pool (3.0.2)
-    docile (1.4.1)
     drb (2.2.3)
     ffi (1.17.4)
     io-console (0.8.2)
-    json (2.19.7)
-    karafka-core (2.5.13)
+    json (2.20.0)
+    karafka-core (2.6.2)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
     karafka-rdkafka (0.27.2)
@@ -39,12 +38,7 @@ GEM
     reline (0.6.3)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.13.2)
-    simplecov_json_formatter (0.1.4)
+    simplecov (1.0.0)
     warning (1.6.0)
     zeitwerk (2.8.2)
 
@@ -65,12 +59,11 @@ DEPENDENCIES
 CHECKSUMS
   byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
-  docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
   ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
-  json (2.19.7) sha256=fe432c8639f6efff69f9d73b518a3705d9581ab93156f981ea72806e1e5bcc3e
-  karafka-core (2.5.13) sha256=0acec083043bb6166c4b647a7458091cc7b08066d3b92a026932925ec7e07f61
+  json (2.20.0) sha256=9362bc6e55a952b056abf9167cf053358181c904cb70cd6eee0808ea830fc32b
+  karafka-core (2.6.2) sha256=c2fd7f277201b8ca97b824b364ad76bf776b8f5527bc422dafb71f7ae48d3a13
   karafka-rdkafka (0.27.2) sha256=3ccce96306642be70bff8168e4e737fc10f2ffae20bc0ff0a43d88dbb7452d31
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
@@ -81,11 +74,9 @@ CHECKSUMS
   rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
   reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
   ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
-  simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
-  simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
-  simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
+  simplecov (1.0.0) sha256=bdc50b41fa4a8b3c860da4bf5d61ba4704479c51d08a8f53cbd56ae50778dca8
   warning (1.6.0) sha256=a49cdfae19fb77d19afff2efbe45f8ab759e9cd25b4e4ce2c79dbaf46bdb6c9e
-  waterdrop (2.10.2)
+  waterdrop (2.10.3)
   zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12
 
 BUNDLED WITH

data/bin/verify_kafka_warnings

@@ -7,6 +7,18 @@ allowed_patterns=(
    "Performing controller activation"
    "registered with feature metadata.version"
    "TOPIC_ALREADY_EXISTS"
+   # The transactional_abort_after_commit_race and transactional_abort_race_canary integration specs
+   # deliberately reproduce librdkafka#4849: they abort with the first produce still in flight, so
+   # the abort's EndTxn can reach the coordinator before AddPartitionsToTxn registers the new
+   # transaction. The coordinator then still holds the *previous* transaction's COMPLETE_COMMIT state
+   # and rejects the ABORT marker, logging this warning. It is the broker-side signature of the very
+   # defect those specs exist to reproduce, so it is expected there - and scoped to their
+   # transactional ids so we still catch the same warning anywhere else.
+   #
+   # One entry per spec rather than an alternation: `\|` is a GNU extension to POSIX BRE, so on BSD
+   # grep (macOS) it would match nothing and the allowance would silently do nothing at all.
+   "tx-abort-race-id.*received transaction marker result to send: ABORT"
+   "tx-abort-canary-id.*received transaction marker result to send: ABORT"
 )
 
 # Get all warnings

data/docker-compose.oauth.yml

@@ -18,7 +18,7 @@ services:
       start_period: 90s
 
   kafka-oauth:
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
     container_name: kafka-oauth
     depends_on:
       keycloak:

data/docker-compose.sasl.yml

@@ -1,6 +1,6 @@
 services:
   kafka-sasl:
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
     container_name: kafka-sasl
     ports:
       - "9095:9095"

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.2.1
+    image: confluentinc/cp-kafka:8.3.0
 
     ports:
       - 9092:9092

data/lib/waterdrop/config.rb

@@ -95,6 +95,35 @@ module WaterDrop
     # option [Integer] How many times to attempt reloading on transactional fatal error before
     #   giving up. This prevents infinite reload loops if the producer never recovers.
     setting :max_attempts_on_transaction_fatal_error, default: 10
+    # option [Numeric] How long to wait **at most** (in ms) for the first delivery of a transaction
+    #   to be acknowledged before we abort that transaction. `0` (the default) disables the wait.
+    #
+    #   This is a ceiling, not a fixed delay: the wait ends as soon as the delivery is acknowledged,
+    #   which under normal conditions is immediate. The full timeout is only ever spent when the
+    #   delivery never arrives at all.
+    #
+    #   This is an opt-in mitigation for a librdkafka defect. librdkafka only flags a transaction as
+    #   ongoing at the coordinator once the `AddPartitionsToTxn` **response** arrives, but it sends
+    #   `EndTxn` as soon as that request has merely been **sent**. Aborting with the first produce
+    #   still in flight can therefore hit a coordinator that does not consider the transaction
+    #   started yet, failing the abort with a fatal `INVALID_TXN_STATE`.
+    #   See https://github.com/confluentinc/librdkafka/issues/4849
+    #
+    #   A single acknowledged delivery proves its partition completed registration, and that alone
+    #   makes the coordinator accept `EndTxn` - no matter how many partitions the transaction spans.
+    #   So waiting for one delivery before aborting closes the race.
+    #
+    #   It is off by default because it changes abort semantics: waiting for that ack means the
+    #   first message is actually **delivered** (aborted, so invisible to `read_committed`
+    #   consumers) instead of being **purged**. Its delivery handle then reports a real offset
+    #   rather than a `Purged in queue` error, and no `message.purged` event is emitted for it,
+    #   while the remaining messages of the same transaction are still purged. Enable this only if
+    #   you hit the defect and prefer that trade over the fatal (which stays recoverable through
+    #   `reload_on_transaction_fatal_error` either way).
+    #
+    #   The wait is bounded and best-effort: if it expires (broker down, message timeout) we abort
+    #   exactly as if it were disabled.
+    setting :wait_timeout_on_transaction_abort, default: 0
     # option [Array<Symbol>] List of fatal error codes that should NOT trigger producer reload.
     #   These errors represent states that cannot be recovered by simply recreating the client.
     #

data/lib/waterdrop/contracts/config.rb

@@ -30,6 +30,7 @@ module WaterDrop
       required(:max_attempts_on_idempotent_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
       required(:wait_backoff_on_transaction_fatal_error) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:max_attempts_on_transaction_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
+      required(:wait_timeout_on_transaction_abort) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:non_reloadable_errors) do |val|
         val.is_a?(Array) && val.all?(Symbol)
       end

data/lib/waterdrop/producer/transactions.rb

@@ -106,9 +106,11 @@ module WaterDrop
             # This is why we catch this here
             begin
               with_transactional_error_handling(:abort) do
-                transactional_instrument(:aborted) do
-                  client.abort_transaction
-                end
+                # Outside of the `aborted` instrumentation on purpose: that event is supposed to
+                # measure `client.abort_transaction`, not our wait
+                transactional_await_first_delivery(e)
+
+                transactional_instrument(:aborted) { client.abort_transaction }
               end
             rescue => e
               # If something from rdkafka leaks here, it means there was a non-retryable error that
@@ -121,6 +123,13 @@ module WaterDrop
             transactional_reload_client_if_needed(e)
 
             raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
+          ensure
+            # The first delivery handle is scoped to the transaction that produced it and is only
+            # ever read by the abort path (which runs in the rescue above, before this). Clearing it
+            # here - on commit, on abort and on a re-raise alike - keeps it from outliving its
+            # transaction: otherwise a committed transaction would leave its handle (and everything
+            # the delivery report references) pinned to the producer until the next one begins.
+            @transaction_first_handle = nil
           end
         end
       end
@@ -255,6 +264,12 @@ module WaterDrop
         if e.abortable? && allow_abortable
           # Always attempt to abort but if aborting fails with an abortable error, do not attempt
           # to abort from abort as this could create an infinite loop
+          #
+          # We deliberately do NOT wait for the first delivery here (unlike the abort in
+          # `#transaction`): we only get here because librdkafka reported an abortable error, so the
+          # transaction is already in an error state where it no longer delivers. The queued messages
+          # are purged by the abort itself, so the handle would never resolve and we would just stall
+          # for the whole `wait_timeout_on_transaction_abort` on a path that is already recovering.
           with_transactional_error_handling(:abort, allow_abortable: false) do
             transactional_instrument(:aborted) { client.abort_transaction }
           end
@@ -263,6 +278,61 @@ module WaterDrop
         raise
       end
 
+      # Waits (bounded) for the first delivery of the current transaction before we abort it.
+      #
+      # librdkafka only marks a transaction as ongoing at the coordinator once the
+      # `AddPartitionsToTxn` **response** comes back, but it fires `EndTxn` as soon as that request
+      # has merely been **sent** (it gates on `txn_req_cnt`, bumped on send). Aborting with the first
+      # produce still in flight can therefore reach a coordinator that does not yet consider the
+      # transaction started, which fails the abort with a fatal `INVALID_TXN_STATE`.
+      # See https://github.com/confluentinc/librdkafka/issues/4849
+      #
+      # A delivered message proves its partition completed registration, and that alone puts the
+      # transaction in an `ongoing` state at the coordinator - so a single acknowledged delivery is
+      # enough to make `EndTxn` valid, regardless of how many partitions the transaction spans.
+      #
+      # This is best-effort and never fatal: if the delivery does not materialize within the timeout
+      # (broker down, message timeout, purge) we abort exactly as before rather than hanging. The
+      # fatal, should it still happen, remains recoverable through the client reload.
+      #
+      # @param error [Exception] the error that is causing us to abort this transaction
+      def transactional_await_first_delivery(error)
+        handle = @transaction_first_handle
+        timeout = config.wait_timeout_on_transaction_abort
+
+        return if handle.nil?
+        return if timeout.zero?
+        # Only worth waiting when the transaction is still healthy, that is when we abort because the
+        # user asked us to (`AbortTransaction`) or because their own code raised. Once librdkafka
+        # itself reported an error, the transaction no longer delivers: the queued messages are
+        # purged by the abort, so this handle would never resolve and we would stall for the full
+        # timeout on a path that is already recovering from an error.
+        return if transactional_error?(error)
+
+        # An already delivered handle (the common case - any sync dispatch, or a transaction that did
+        # some work before aborting) short-circuits inside `#wait` itself, so there is nothing to
+        # guard against here: it returns immediately without blocking.
+        wait(handle, max_wait_timeout: timeout, raise_response_error: false)
+      rescue ::Rdkafka::AbstractHandle::WaitTimeoutError, ::Rdkafka::RdkafkaError
+        # These two mean the same thing for us: we could not confirm the registration, either because
+        # the delivery did not arrive in time or because it failed outright. Aborting is still the
+        # right move, we just lose the mitigation for this one transaction.
+        #
+        # Deliberately narrow. A broader rescue would also swallow bugs of our own making (a
+        # `NoMethodError` on a handle we mis-tracked, say) and turn them into a silently skipped
+        # mitigation that nobody would ever notice.
+        nil
+      end
+
+      # @param error [Exception] error that caused the abort
+      # @return [Boolean] did this error come from librdkafka, meaning the transaction is already in
+      #   an error state and will not deliver anything anymore
+      def transactional_error?(error)
+        return true if error.is_a?(::Rdkafka::RdkafkaError)
+
+        error.cause.is_a?(::Rdkafka::RdkafkaError)
+      end
+
       # Reloads the underlying client instance if needed and allowed
       #
       # This should be used only in transactions as only then we can get fatal transactional

data/lib/waterdrop/producer.rb

@@ -67,6 +67,7 @@ module WaterDrop
       @poller = nil
       @idempotent_fatal_error_attempts = 0
       @transaction_fatal_error_attempts = 0
+      @transaction_first_handle = nil
 
       @status = Status.new
       @messages = []
@@ -647,6 +648,12 @@ module WaterDrop
         client.produce(**message)
       end
 
+      # Remember the first delivery handle of the current transaction. Aborting while the very first
+      # produce is still in flight is what triggers librdkafka#4849, so the abort path waits on this
+      # handle to confirm the transaction is registered at the coordinator. See
+      # `#transactional_await_first_delivery`.
+      @transaction_first_handle ||= result if transactional? && @transaction_mutex.owned?
+
       # Reset attempts counter on successful produce
       @idempotent_fatal_error_attempts = 0
 

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = "2.10.2"
+  VERSION = "2.10.3"
 end

data/package-lock.json

@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz",
-      "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==",
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz",
+      "integrity": "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.10.2
+  version: 2.10.3
 platform: ruby
 authors:
 - Maciej Mensfeld