waterdrop 2.10.4 → 2.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32f1c3b166f74db06dc75080800f4972207facc943d00d7eeab73a6c33668aed
-  data.tar.gz: 8de1c8eecf566e241bc375fea7de5f568cd055be35a83effcf5de23b7c92f468
+  metadata.gz: df6f8bc25170845840699e442886688586d193d69ddb89849bfbd0a6da2c7dbd
+  data.tar.gz: 15cd50dfc2d196580d5afb44aef4ecab2683b7ce11f966d06041c4f5f120cbfc
 SHA512:
-  metadata.gz: 901c6f6a576c38807eee47641675b12f9ef3907325fb2435c6b2e8656724c4f57be6593a10b7b85348b8cadd317216fe84ba182a9ccfaf3fa0208119c0609686
-  data.tar.gz: 25720c9a6ac31520400f181af47b389eae9def35f384cf9b1d89f8466c4cafc904c0390fbe14a1222c6917b3b26fbcc38d9580e9a0fa45db07d1cc9528c3bb85
+  metadata.gz: 6a0dd37309d89f9ffa8d3932f338edf56f059bbd620befc1c008c6375c258f515e4aec054370815fc56411ba787f887ca9bf7bc5173ec7cf6117f6db44cec7ce
+  data.tar.gz: 87e75e5ec48f647bc070603d49848a9d5032ab89fb411ac4e33683311b70736e287de5d6cdfebb4c1a77c96d7f09438e9c85c597faea7673031e11ff64f25bfa

data/.rubocop.yml

@@ -12,7 +12,7 @@ inherit_gem:
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 3.2
+  TargetRubyVersion: 3.3
   Include:
     - "**/*.rb"
     - "**/*.gemspec"

data/.ruby-version

@@ -1 +1 @@
-4.0.6
+4.0.7

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # WaterDrop changelog
 
+## 2.10.6 (2026-10-12)
+- **[EOL]** Drop Ruby 3.2 support; require Ruby 3.3+.
+- [Maintenance] Document that a middleware chain run is not atomic, so an in-place step followed by a raising step is applied again on the next `#flush`.
+- [Maintenance] Stop the sync delivery error spec from flaking when librdkafka rejects the produce to the invalid topic inline instead of through the delivery report.
+
+## 2.10.5 (2026-09-23)
+- [Fix] Stop `#flush` from running middleware twice on messages re-buffered after a failed flush, which double-processed payloads and duplicated headers on retry (regression of #474).
+- [Fix] Raise `ProducerClosedError` instead of silently losing a message buffered while the producer is closing.
+- [Fix] Restore both buffers when `#flush` fails before anything is dispatched, instead of discarding the whole batch.
+
 ## 2.10.4 (2026-08-26)
 - [Fix] Avoid a `FrozenError` on `Producer#close` when the producer is configured with a frozen string id (for example a frozen string literal, `config.id = "rspec"`).
 - [Fix] Emit `connection_pool.shutdown` only once when shutting down the global connection pool.
@@ -7,10 +17,10 @@
 - [Maintenance] Stop the `#partition_count when topic does not exist` spec from flaking on slow CI runners by waiting for authoritative broker metadata before asserting the count.
 
 ## 2.10.3 (2026-07-15)
-- [Feature] Add `wait_timeout_on_transaction_abort` (default `0`, disabled) - an opt-in mitigation for [librdkafka#4849](https://github.com/confluentinc/librdkafka/issues/4849) that waits (up to the given ms) for the first delivery to be acknowledged before aborting, avoiding a fatal `INVALID_TXN_STATE`. Off by default because it changes abort semantics: the awaited message is written to the log (aborted) instead of being purged.
+- [Feature] Add `wait_timeout_on_transaction_abort` (disabled by default) to avoid a fatal `INVALID_TXN_STATE` on abort ([librdkafka#4849](https://github.com/confluentinc/librdkafka/issues/4849)). When enabled, the awaited message is delivered as aborted instead of purged.
 
 ## 2.10.2 (2026-06-15)
-- [Feature] Expose `Producer#current_variant` as a public method returning the variant active for the current dispatch on the current fiber, so middleware and instrumentation listeners can read the effective per-dispatch settings (`topic_config`, `max_wait_timeout`, `default?`).
+- [Feature] Expose `Producer#current_variant`, returning the variant active for the current dispatch, for use in middleware and instrumentation listeners.
 - [Enhancement] Stop allocating one interpolated string per message in `LoggerListener` batch produce handlers.
 - [Enhancement] Use `Array#concat` in `Producer#buffer_many` instead of appending messages one by one.
 - [Enhancement] Skip building the `message.acknowledged` instrumentation payload in the delivery callback when nothing is subscribed to that event.
@@ -19,22 +29,22 @@
 - [Enhancement] Cache the variant validation contract in a constant instead of instantiating a new `Contracts::Variant` on every `Producer#with` / `Producer#variant` call.
 - [Enhancement] Cache the tombstone validation contract in a constant instead of instantiating a new `Contracts::Tombstone` per tombstone message.
 - [Enhancement] Replace explicit `Warning[:performance]` opt-in with a dynamic approach using `Warning.categories` to enable all stable opt-in warning categories in the test suite.
-- [Fix] Prevent a deadlock between a transactional single-message dispatch and `#close` caused by an inverted lock order; transactional dispatches now take `@transaction_mutex` before the operation is counted.
-- [Fix] Prevent a deadlock (`ThreadError: deadlock; recursive locking`) when closing an idempotent producer (with `reload_on_idempotent_fatal_error` enabled) whose final buffer flush surfaces a fatal librdkafka error; the idempotent reload is now skipped on the closing path.
+- [Fix] Prevent a deadlock between a transactional single-message dispatch and `#close`.
+- [Fix] Prevent a deadlock when closing an idempotent producer with `reload_on_idempotent_fatal_error` enabled whose final flush hits a fatal error.
 - [Fix] Make concurrent idempotent fatal-error reload thread-safe so a second thread's reload can no longer raise `NoMethodError` after the first reset `@client`.
 - [Fix] Stop `#flush_async` / `#flush_sync` from silently dropping valid buffered messages when the dispatch fails; unsent messages are now re-buffered so they can be retried instead of being lost.
 - [Fix] Make `Producer#close` fork-safe so the GC finalizer inherited by a forked child can no longer flush and close the parent's client.
-- [Fix] Guard the internal buffer appends in `Producer#buffer` and `Producer#buffer_many` with `@buffer_mutex` so a concurrent flush/purge/close swap can no longer drop a message into an orphaned array.
-- [Fix] Stop a nested same-producer variant call from clobbering the outer variant inside a variant `transaction` block; the wrapper now saves and restores the previous fiber-local entry instead of deleting it.
-- [Fix] Stop `ConnectionPool#shutdown` and `#reload` from silently dropping in-flight messages. They now close producers gracefully by default (`#reload` always; `#shutdown` unless called with the new `force: true`); pass `pool.shutdown(force: true)` to keep the old force-and-purge behavior.
-- [Fix] Close a race in the FD poller where a producer registered while the last one was being torn down could be left permanently unpolled; the poller now decides to stop and clears its thread reference in a single mutex section.
+- [Fix] Prevent a concurrent flush, purge or close from dropping a message buffered with `#buffer` or `#buffer_many`.
+- [Fix] Stop a nested variant call on the same producer from overriding the outer variant inside a variant `transaction` block.
+- [Fix] Stop `ConnectionPool#shutdown` and `#reload` from dropping in-flight messages by closing producers gracefully. Use `pool.shutdown(force: true)` for the previous force-and-purge behavior.
+- [Fix] Fix a race that could leave a newly registered producer permanently unpolled by the FD poller.
 
 ## 2.10.1 (2026-05-25)
-- [Fix] Prevent `Producer#close` from raising `ThreadError: can't be called from trap context` when invoked from a Ruby signal trap context (e.g. Puma's `after_stopped` DSL hook in single mode). `close` now detects this case and delegates to a background thread, joining it so the caller blocks until the producer is fully closed (#866).
+- [Fix] Prevent `Producer#close` from raising `ThreadError: can't be called from trap context` when called from a signal trap (e.g. Puma's `after_stopped` hook in single mode). It now closes from a background thread and waits for it (#866).
 
 ## 2.10.0 (2026-05-07)
-- [Fix] Clean up native rdkafka client, global instrumentation callbacks, and poller registration when `init_transactions` fails during producer client construction, so failed attempts no longer leak native threads, file descriptors, and callback registry entries.
-- **[Breaking]** Skip emitting librdkafka statistics when nothing is subscribed to `statistics.emitted` at the time the underlying rdkafka client is constructed (`statistics.interval.ms` is forced to `0`), saving substantial allocations in the hot path. To use statistics, subscribe a listener to `statistics.emitted` BEFORE the first producer use.
+- [Fix] Clean up the native client, instrumentation callbacks and poller registration when `init_transactions` fails during producer construction, so failed attempts no longer leak threads and file descriptors.
+- **[Breaking]** Skip librdkafka statistics when nothing subscribes to `statistics.emitted` before the rdkafka client is built, to save allocations. To use statistics, subscribe a listener BEFORE the first producer use.
 - **[Breaking]** Raise `WaterDrop::Errors::StatisticsNotEnabledError` when attempting to subscribe to `statistics.emitted` on a monitor where librdkafka statistics have been disabled at client build time, replacing the previous "silent nothing" failure mode.
 - [Feature] Add tombstone API (`#tombstone_sync`, `#tombstone_async`, `#tombstone_many_sync`, `#tombstone_many_async`) for producing tombstone records (nil-payload messages) with required key and partition validation. Works with variants.
 - [Fix] Add `ensure_same_process!` to `Poller#unregister` for fork safety, so a child that inherited a pre-fork producer no longer deadlocks on `producer.close`.
@@ -43,14 +53,14 @@
 - [Fix] Use `delete` in the variant ensure block to avoid leaving stale nil entries in `Fiber.current.waterdrop_clients` and prevent memory leaks in long-running processes (#836).
 - [Fix] Exclude test files, `.github/`, and `log/` directories from gem releases to reduce package size.
 - **[Breaking]** Switch default polling mode from `:thread` to `:fd`. If you experience any issues, you can revert to the previous behavior by setting `config.polling.mode = :thread`. The `:thread` mode will be deprecated in 2.10 and removed in 2.11.
-- **[Breaking]** Statistics decorator now only decorates keys used by the built-in Datadog metrics listener (`tx`, `txretries`, `txerrs`, `rxerrs`) and skips unused subtrees, greatly reducing decoration cost on large clusters. Users who rely on other `_d` or `_fd` keys in custom instrumentation should provide a custom decorator via `config.statistics_decorator`.
+- **[Breaking]** Decorate only the statistics keys used by the built-in Datadog listener (`tx`, `txretries`, `txerrs`, `rxerrs`). If you use other `_d` or `_fd` keys, set a custom `config.statistics_decorator`.
 - [Feature] Add `config.statistics_decorator` setting to allow full control over the `StatisticsDecorator` instance used for statistics decoration. Users can provide a custom decorator with different `only_keys` and `excluded_keys` to match their instrumentation needs.
 - [Change] Upscale default timeout values 3x closer to librdkafka defaults to prevent intermediate timeouts during node recovery (`message.timeout.ms`: 50s → 150s, `transaction.timeout.ms`: 55s → 165s, `max_wait_timeout`: 60s → 180s).
 
 **Upgrade Notes**: https://karafka.io/docs/Upgrades-WaterDrop-2.9/
 
 ## 2.8.16 (2026-02-25)
-- [Feature] Add FD-based polling mode (`config.polling.mode = :fd`) as an alternative to the default thread-based polling. FD mode uses a single Ruby thread with IO.select for efficient multiplexing, providing 39-54% higher throughput, lower memory usage, and fewer threads compared to the default thread mode.
+- [Feature] Add FD-based polling (`config.polling.mode = :fd`): a single Ruby thread with `IO.select` instead of the default thread mode, with 39-54% higher throughput, lower memory usage and fewer threads.
 - [Feature] Add `config.polling.fd.max_time` setting (default: 100ms) to control maximum polling time per producer per cycle. This enables per-producer priority differentiation.
 - [Feature] Add `Producer#queue_size` (alias: `#queue_length`) to return the count of messages pending in the librdkafka queue. This counts messages that have been dispatched to librdkafka but not yet transmitted to the Kafka broker.
 - [Enhancement] Add `poll_nb` and `poll_drain` methods to rdkafka Producer for efficient non-blocking polling without GVL release overhead.
@@ -77,11 +87,7 @@
 
 ## 2.8.12 (2025-10-10)
 - [Enhancement] Introduce `reload_on_idempotent_fatal_error` to automatically reload librdkafka producer after fatal errors on idempotent (non-transactional) producers.
-- [Enhancement] Add configurable backoff and retry limits for fatal error recovery to prevent infinite reload loops:
-  - `wait_backoff_on_idempotent_fatal_error` (default: 5000ms) - backoff before retrying after idempotent fatal error reload
-  - `max_attempts_on_idempotent_fatal_error` (default: 5) - max reload attempts for idempotent fatal errors
-  - `wait_backoff_on_transaction_fatal_error` (default: 1000ms) - backoff after transactional fatal error reload
-  - `max_attempts_on_transaction_fatal_error` (default: 10) - max reload attempts for transactional fatal errors
+- [Enhancement] Add backoff and retry limits for fatal error reloads to prevent infinite reload loops (defaults in brackets): `wait_backoff_on_idempotent_fatal_error` (5000ms), `max_attempts_on_idempotent_fatal_error` (5), `wait_backoff_on_transaction_fatal_error` (1000ms) and `max_attempts_on_transaction_fatal_error` (10).
 - [Enhancement] Ensure `error.occurred` is instrumented before idempotent fatal error reload for visibility.
 - [Enhancement] Automatically reset fatal error reload attempts counter on successful produce/transaction to allow recovery.
 - [Refactor] Extract idempotence-related logic into separate `WaterDrop::Producer::Idempotence` module.
@@ -107,7 +113,7 @@
 
 ## 2.8.8 (2025-09-23)
 - [Feature] Add `WaterDrop::ConnectionPool` for efficient connection pooling using the proven `connection_pool` gem.
-- [Feature] Add `WaterDrop.instrumentation` class-level instrumentation for producer lifecycle events. This allows external libraries to subscribe to `producer.created` and `producer.configured` events without needing producer instance references, enabling middleware injection and configuration by libraries like Datadog tracing.
+- [Feature] Add class-level `WaterDrop.instrumentation` with `producer.created` and `producer.configured` events, so libraries (e.g. Datadog tracing) can configure producers without instance references.
 - **[EOL]** Remove Ruby `3.1` specs according to the EOL schedule.
 
 ## 2.8.7 (2025-09-02)
@@ -196,7 +202,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - **[Breaking]** Remove no longer needed `wait_timeout` configuration option.
 - **[Breaking]** Do **not** validate or morph (via middleware) messages added to the buffer prior to `flush_sync` or `flush_async`.
 - [Enhancement] Provide `WaterDrop::Producer#transaction?` that returns only when producer has an active transaction running.
-- [Enhancement] Introduce `instrument_on_wait_queue_full` flag (defaults to `true`) to be able to configure whether non critical (retryable) queue full errors should be instrumented in the error pipeline. Useful when building high-performance pipes with WaterDrop queue retry backoff as a throttler.
+- [Enhancement] Introduce `instrument_on_wait_queue_full` (defaults to `true`) to control whether retryable queue full errors are instrumented in the error pipeline.
 - [Enhancement] Protect critical `rdkafka` thread executable code sections.
 - [Enhancement] Treat the queue size as a gauge rather than a cumulative stat (isturdy).
 - [Fix] Fix a case where purge on non-initialized client would crash.
@@ -220,7 +226,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - [Fix] Fix a case where buffered test client would not accumulate messages on failed transactions
 
 ## 2.6.10 (2023-10-24)
-- [Improvement] Introduce `message.purged` event to indicate that a message that was not delivered to Kafka was purged. This most of the time refers to messages that were part of a transaction and were not yet dispatched to Kafka. It always means, that given message was not delivered but in case of transactions it is expected. In case of non-transactional it usually means `#purge` usage or exceeding `message.timeout.ms` so `librdkafka` removes this message from its internal queue. Non-transactional producers do **not** use this and pipe purges to `error.occurred`.
+- [Improvement] Introduce the `message.purged` event for a transactional message purged before delivery to Kafka. Non-transactional producers report purges via `error.occurred` instead.
 - [Fix] Fix a case where `message.acknowledged` would not have `caller` key.
 - [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
 
@@ -274,7 +280,7 @@ This release contains **BREAKING** changes. Make sure to read and apply upgrade
 - [Improvement] Make `#produce` method private to avoid confusion and make sure it is not used directly (it is not part of the official API).
 - [Change] Change `wait_on_queue_full` from `false` to `true` as a default.
 - [Change] Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full` to match what it actually does.
-- [Enhancement] Introduce `wait_timeout_on_queue_full` with proper meaning. That is, this represents time after which despite backoff the error will be raised. This should allow to raise an error in case the backoff attempts were insufficient. This prevents from a case, where upon never deliverable messages we would end up with an infinite loop.
+- [Enhancement] Introduce `wait_timeout_on_queue_full`: the time after which the error is raised despite backoff, so never-deliverable messages no longer cause an infinite loop.
 - [Fix] Provide `type` for queue full errors that references the appropriate public API method correctly.
 
 ## 2.5.3 (2023-05-26)

data/Gemfile

@@ -4,7 +4,6 @@ source "https://rubygems.org"
 
 gemspec
 
-# Relaxed from 2.7 because we support Ruby 3.1
 gem "zeitwerk", "~> 2.8.0"
 
 group :development do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.10.4)
+    waterdrop (2.10.6)
       karafka-core (>= 2.5.12, < 3.0.0)
       karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
@@ -14,12 +14,12 @@ GEM
     connection_pool (3.0.2)
     drb (2.2.3)
     ffi (1.17.4)
-    io-console (0.8.2)
-    json (2.21.2)
-    karafka-core (2.6.2)
+    io-console (0.9.4)
+    json (3.0.2)
+    karafka-core (2.6.4)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
-    karafka-rdkafka (0.28.0)
+    karafka-rdkafka (0.30.2)
       ffi (~> 1.17.1)
       json (> 2.0)
       logger
@@ -35,10 +35,10 @@ GEM
     ostruct (0.6.3)
     prism (1.9.0)
     rake (13.4.2)
-    reline (0.6.3)
+    reline (0.7.0)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    simplecov (1.1.1)
+    simplecov (1.3.2)
     warning (1.6.0)
     zeitwerk (2.8.3)
 
@@ -61,10 +61,10 @@ CHECKSUMS
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
   ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
-  io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
-  json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a
-  karafka-core (2.6.2) sha256=c2fd7f277201b8ca97b824b364ad76bf776b8f5527bc422dafb71f7ae48d3a13
-  karafka-rdkafka (0.28.0) sha256=fd845140c42341c5892215cac1e8adacda53b0461688f2e16379d82cb33eb006
+  io-console (0.9.4) sha256=a403304f32928ee85e56d457b4612ca43f2703b561a3258844992bd24085fbb1
+  json (3.0.2) sha256=8e6d7e7b11384c21230430cef90b71f14849a34a1f4452796670f7c981bd19df
+  karafka-core (2.6.4) sha256=e08166ca7ef12fadf0500f2a3aba63b5ba918a98533c3feaac9f43c90f159363
+  karafka-rdkafka (0.30.2) sha256=11d565109ab9ebc6216020d6097bd9adbc39a9e599fb2a8cc5af93965b9008b7
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
   minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1
@@ -72,11 +72,11 @@ CHECKSUMS
   ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
   rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
-  reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
+  reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d
   ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
-  simplecov (1.1.1) sha256=25825ef13f0b2e74694d769817dad6ab8e90131dabdaa666e522fea105521e78
+  simplecov (1.3.2) sha256=2a22ec1759f1b8db60881a2e038758956fc69ef93ed669be0bde0678de6d0e85
   warning (1.6.0) sha256=a49cdfae19fb77d19afff2efbe45f8ab759e9cd25b4e4ce2c79dbaf46bdb6c9e
-  waterdrop (2.10.4)
+  waterdrop (2.10.6)
   zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5
 
 BUNDLED WITH

data/README.md

@@ -1,7 +1,7 @@
 # WaterDrop
 
-[![Build Status](https://github.com/karafka/waterdrop/workflows/ci/badge.svg)](https://github.com/karafka/waterdrop/actions?query=workflow%3Aci)
-[![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
+[![Build Status](https://github.com/karafka/waterdrop/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/waterdrop/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](https://badge.fury.io/rb/waterdrop)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
 WaterDrop is a standalone gem that sends messages to Kafka easily with an extra validation layer. It is a part of the [Karafka](https://github.com/karafka/karafka) ecosystem.
@@ -16,7 +16,7 @@ It:
  - Supports multiple delivery policies
  - Supports per-topic configuration alterations (variants)
  - Works with [async](https://github.com/socketry/async) gems ecosystem
- - Works with Kafka `1.0+` and Ruby `3.2+`
+ - Works with Kafka `1.0+` and Ruby `3.3+`
  - Works with and without Karafka
 
 ## Documentation

data/bin/verify_kafka_warnings

@@ -4,6 +4,11 @@
 # Only specified warnings are allowed, all others should trigger failure
 
 allowed_patterns=(
+   # Enabling share groups (KIP-932) requires "share" in group.coordinator.rebalance.protocols
+   # on cp-kafka 8.3.0 (Kafka 4.x). That mechanism is deprecated in favour of the share.version
+   # feature flag (Kafka 5.0), but is still functionally required to enable share groups in this
+   # version, so the broker logs this deprecation warning on every startup. Expected here.
+   "group.coordinator.rebalance.protocols. is deprecated"
    "Performing controller activation"
    "registered with feature metadata.version"
    "TOPIC_ALREADY_EXISTS"

data/docker-compose.oauth.yml

@@ -49,6 +49,9 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       # Required by Confluent image when SASL is enabled
       # Also allow HTTP URLs for JWKS endpoint (Kafka 3.0+ blocks non-HTTPS by default)
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf -Dorg.apache.kafka.sasl.oauthbearer.allowed.urls=http://keycloak:8080/realms/kafka/protocol/openid-connect/certs"

data/docker-compose.sasl.yml

@@ -21,6 +21,9 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
       KAFKA_OPTS: ""

data/docker-compose.yml

@@ -21,5 +21,8 @@ services:
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer

data/lib/waterdrop/middleware.rb

@@ -17,6 +17,14 @@ module WaterDrop
     #   into a new object.
     # @note You need to decide yourself whether you don't use the message hash data anywhere else
     #   and you want to save on memory by modifying it in place or do you want to do a deep copy
+    # @note Steps run sequentially and a chain run is not atomic. If an in-place step mutates the
+    #   message and a later step raises, the message is left partially transformed. During a
+    #   buffer flush that message is re-buffered as not yet processed, so the next flush runs the
+    #   whole chain over it again and the earlier in-place steps are applied twice (for example
+    #   double encryption or duplicated headers). Making the run atomic would mean copying every
+    #   message before its chain runs, which defeats the memory saving in-place mode is for, so
+    #   this is a deliberate trade-off. If an in-place step can be followed by a step that may
+    #   raise, make the in-place step idempotent or have it return a copy instead.
     def run(message)
       return message if @count.zero?
 

data/lib/waterdrop/producer/async.rb

@@ -44,15 +44,18 @@ module WaterDrop
       #
       # @param messages [Array<Hash>] array with messages that comply with the
       #   {Contracts::Message} contract
+      # @param run_middleware [Boolean] should the middleware chain be run on the messages. Set to
+      #   false only when the messages have already been through middleware (for example when
+      #   re-dispatching a re-buffered flush batch) so it is not applied twice.
       #
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
       #
       # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
-      def produce_many_async(messages)
+      def produce_many_async(messages, run_middleware: true)
         dispatched = []
-        messages = middleware.run_many(messages)
+        messages = middleware.run_many(messages) if run_middleware
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/buffer.rb

@@ -15,12 +15,22 @@ module WaterDrop
         # The append runs under @buffer_mutex because flush/purge/close swap @messages for a fresh
         # array under the same lock. Without it, a concurrent swap between reading @messages and
         # appending would land the message in the orphaned old array and silently lose it.
+        #
+        # The liveness check is repeated under the same lock because `close` marks the producer
+        # `:closing` before its final flush acquires @buffer_mutex. Checking only above the lock
+        # would let a close complete in between, stranding the message in a closed producer's
+        # buffer while this call returned success.
         @monitor.instrument(
           "message.buffered",
           producer_id: id,
           message: message,
           buffer: @messages
-        ) { @buffer_mutex.synchronize { @messages << message } }
+        ) do
+          @buffer_mutex.synchronize do
+            ensure_active!
+            @messages << message
+          end
+        end
       end
 
       # Adds given messages into the internal producer buffer without flushing them to Kafka
@@ -34,14 +44,20 @@ module WaterDrop
 
         # The concat runs under @buffer_mutex for the same reason as #buffer: flush/purge/close swap
         # @messages under the lock, so an unguarded concat could append into an array that has just
-        # been captured for dispatch (or discarded), silently losing the messages.
+        # been captured for dispatch (or discarded), silently losing the messages. The liveness
+        # check is repeated under the lock for the same reason as in #buffer: a close that
+        # completes between the check above and the lock would otherwise strand the batch.
         @monitor.instrument(
           "messages.buffered",
           producer_id: id,
           messages: messages,
           buffer: @messages
         ) do
-          @buffer_mutex.synchronize { @messages.concat(messages) }
+          @buffer_mutex.synchronize do
+            ensure_active!
+            @messages.concat(messages)
+          end
+
           messages
         end
       end
@@ -53,7 +69,7 @@ module WaterDrop
         @monitor.instrument(
           "buffer.flushed_async",
           producer_id: id,
-          messages: @messages
+          messages: messages
         ) { flush(false) }
       end
 
@@ -64,7 +80,7 @@ module WaterDrop
         @monitor.instrument(
           "buffer.flushed_sync",
           producer_id: id,
-          messages: @messages
+          messages: messages
         ) { flush(true) }
       end
 
@@ -78,17 +94,53 @@ module WaterDrop
       # @note We use this method underneath to provide a different instrumentation for sync and
       #   async flushing within the public API
       def flush(sync)
-        data_for_dispatch = nil
+        fresh = nil
+        requeued = nil
+        data_for_dispatch = []
 
         @buffer_mutex.synchronize do
-          data_for_dispatch = @messages
+          requeued = @requeued
+          @requeued = []
+          fresh = @messages
           @messages = []
         end
 
+        # Middleware runs exactly once per message: on the fresh messages here, never again on the
+        # already-transformed ones coming back from a failed flush, which a second pass would
+        # corrupt.
+        #
+        # A middleware step raising here (a serializer, encryptor or schema lookup failing) would
+        # otherwise destroy both buffers, since they were emptied above and nothing has been
+        # dispatched yet to recover them from. Messages are run one by one so that on failure the
+        # ones whose chain completed are kept as returned and are not transformed again, while the
+        # failing one and those after it stay on the middleware path. The failing one is not
+        # requeued because that would dispatch it without the rest of its chain.
+        #
+        # Known edge case (#46840): the failing message is re-buffered in whatever state its chain
+        # left it. Copy-style steps leave it untouched, but in-place steps that ran before the
+        # raising one leave it partially transformed, and the next flush runs its full chain again.
+        # We accept this rather than snapshotting every message before its run, which would cost a
+        # copy per message on the success path as well. See Middleware#run.
+        processed = []
+
+        begin
+          fresh.each { |message| processed << middleware.run(message) }
+        rescue
+          restore_undispatched(fresh.drop(processed.size), requeued.concat(processed))
+
+          raise
+        end
+
+        data_for_dispatch = requeued.concat(processed)
+
         # Do nothing if nothing to flush
         return data_for_dispatch if data_for_dispatch.empty?
 
-        sync ? produce_many_sync(data_for_dispatch) : produce_many_async(data_for_dispatch)
+        if sync
+          produce_many_sync(data_for_dispatch, run_middleware: false)
+        else
+          produce_many_async(data_for_dispatch, run_middleware: false)
+        end
       rescue Errors::ProduceManyError => e
         # A dispatch failed partway through the batch. Re-buffer the messages that never reached
         # librdkafka so a partial failure does not silently drop valid buffered messages. For a
@@ -106,15 +158,27 @@ module WaterDrop
         raise
       end
 
-      # Puts not-yet-dispatched messages back at the front of the buffer (preserving their original
-      # order relative to each other and to anything buffered concurrently), so a failed flush does
-      # not lose them.
+      # Puts both sets back after a failure that happened before anything reached librdkafka. They
+      # go to different buffers because middleware has run on one of them and not the other, which
+      # is what keeps it applied exactly once per message.
+      #
+      # @param fresh [Array<Hash>] messages whose middleware chain has not completed
+      # @param requeued [Array<Hash>] already middleware-processed messages
+      def restore_undispatched(fresh, requeued)
+        @buffer_mutex.synchronize do
+          @messages.unshift(*fresh)
+          @requeued.unshift(*requeued)
+        end
+      end
+
+      # Puts not-yet-dispatched messages back at the front of the retry buffer (preserving their
+      # original order relative to each other), so a failed flush does not lose them.
       #
-      # @param messages [Array<Hash>] messages to restore to the buffer
+      # @param messages [Array<Hash>] already middleware-processed messages to restore
       def requeue_unflushed(messages)
         return if messages.empty?
 
-        @buffer_mutex.synchronize { @messages.unshift(*messages) }
+        @buffer_mutex.synchronize { @requeued.unshift(*messages) }
       end
     end
   end

data/lib/waterdrop/producer/sync.rb

@@ -48,6 +48,9 @@ module WaterDrop
       #
       # @param messages [Array<Hash>] array with messages that comply with the
       #   {Contracts::Message} contract
+      # @param run_middleware [Boolean] should the middleware chain be run on the messages. Set to
+      #   false only when the messages have already been through middleware (for example when
+      #   re-dispatching a re-buffered flush batch) so it is not applied twice.
       #
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
       #   sent (can be used to verify offset, partition, etc via `#create_result`)
@@ -63,8 +66,8 @@ module WaterDrop
       #   errors), returning handles guarantees the same type in the `dispatched` collection,
       #   allowing uniform error handling. Each handle is in its final state after this method
       #   returns, so you can call `handle.create_result` to obtain the delivery report if needed.
-      def produce_many_sync(messages)
-        messages = middleware.run_many(messages)
+      def produce_many_sync(messages, run_middleware: true)
+        messages = middleware.run_many(messages) if run_middleware
         messages.each { |message| validate_message!(message) }
 
         dispatched = []

data/lib/waterdrop/producer.rb

@@ -39,8 +39,6 @@ module WaterDrop
     attr_reader :id
     # @return [Status] producer status object
     attr_reader :status
-    # @return [Array] internal messages buffer
-    attr_reader :messages
     # @return [Object] monitor we want to use
     attr_reader :monitor
     # @return [Object] dry-configurable config object
@@ -71,6 +69,9 @@ module WaterDrop
 
       @status = Status.new
       @messages = []
+      # Messages re-buffered after a failed flush. Kept apart from @messages because they have
+      # already passed through middleware and must not be transformed again on the next flush.
+      @requeued = []
 
       # Instrument producer creation for global listeners
       class_monitor.instrument(
@@ -177,6 +178,12 @@ module WaterDrop
       @client
     end
 
+    # @return [Array] internal messages buffer awaiting a flush. Includes both freshly buffered
+    #   messages and any messages re-buffered after a failed flush that are waiting to be retried.
+    def messages
+      @requeued + @messages
+    end
+
     # Returns the number of messages in the librdkafka producer queue.
     #
     # This count includes:
@@ -232,6 +239,7 @@ module WaterDrop
       @monitor.instrument("buffer.purged", producer_id: id) do
         @buffer_mutex.synchronize do
           @messages = []
+          @requeued = []
         end
 
         # We should not purge if there is no client initialized
@@ -321,6 +329,7 @@ module WaterDrop
             return false unless @client
             return false unless @status.connected?
             return false unless @messages.empty?
+            return false unless @requeued.empty?
             return false unless @operations_in_progress.value.zero?
 
             @status.disconnecting!
@@ -354,6 +363,7 @@ module WaterDrop
       return false unless @client
       return false unless @status.connected?
       return false unless @messages.empty?
+      return false unless @requeued.empty?
       return false if @transaction_mutex.locked?
       return false if @operating_mutex.locked?
 
@@ -503,7 +513,7 @@ module WaterDrop
       # Try to get buffer info safely
       if @buffer_mutex.try_lock
         begin
-          parts << "buffer_size=#{@messages.size}"
+          parts << "buffer_size=#{@messages.size + @requeued.size}"
         ensure
           @buffer_mutex.unlock
         end

data/lib/waterdrop/version.rb

@@ -2,5 +2,5 @@
 
 module WaterDrop
   # Current WaterDrop version
-  VERSION = "2.10.4"
+  VERSION = "2.10.6"
 end

data/package-lock.json

@@ -103,9 +103,9 @@
       }
     },
     "node_modules/fastq": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
-      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
+      "version": "1.20.3",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
+      "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
       "dev": true,
       "license": "ISC",
       "dependencies": {
@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.7.1",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
-      "integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==",
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
+      "integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {
@@ -312,9 +312,9 @@
       }
     },
     "node_modules/yaml": {
-      "version": "2.9.0",
-      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
-      "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
+      "version": "2.9.1",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
+      "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
       "dev": true,
       "license": "ISC",
       "bin": {

data/renovate.json

@@ -1,4 +1,7 @@
 {
+  "bundler": {
+    "managerFilePatterns": ["/(^|/)Gemfile\\.lint$/"]
+  },
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
     "config:recommended"

data/waterdrop.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
   spec.add_dependency "zeitwerk", "~> 2.3"
 
-  spec.required_ruby_version = ">= 3.2.0"
+  spec.required_ruby_version = ">= 3.3.0"
 
   spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match?(%r{^(spec|test|\.github|log)/}) }
   spec.executables = []

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.10.4
+  version: 2.10.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -153,14 +153,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Kafka messaging made easy!
 test_files: []