waterdrop 2.10.4 → 2.10.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32f1c3b166f74db06dc75080800f4972207facc943d00d7eeab73a6c33668aed
-  data.tar.gz: 8de1c8eecf566e241bc375fea7de5f568cd055be35a83effcf5de23b7c92f468
+  metadata.gz: d61c97700e2882107b4e3309176c22fc85d06df909e3f11f0779bb33cab0ec96
+  data.tar.gz: 1b4c0e3c7ac54d6b86905544396865934ee783ba730dc0ccc6b3bd6b954a7ce8
 SHA512:
-  metadata.gz: 901c6f6a576c38807eee47641675b12f9ef3907325fb2435c6b2e8656724c4f57be6593a10b7b85348b8cadd317216fe84ba182a9ccfaf3fa0208119c0609686
-  data.tar.gz: 25720c9a6ac31520400f181af47b389eae9def35f384cf9b1d89f8466c4cafc904c0390fbe14a1222c6917b3b26fbcc38d9580e9a0fa45db07d1cc9528c3bb85
+  metadata.gz: 9d509ae8aafda829f217357e6d8447905910137a4af850cd8d4bac7a35bb3af703432339c05fa45117dd8acff2a481718f18a94dd7e7ea0ba0259a86b6ae8d42
+  data.tar.gz: 4a8c01eb9aa11c49eb810888a65a520731298ea500bdcaa2c0b78bc163ffc346cc3975f8cad06ea7e50c9c9d21357457eaf745269b35e598a9eccebaf611e7a4

data/.ruby-version

@@ -1 +1 @@
-4.0.6
+4.0.7

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # WaterDrop changelog
 
+## 2.10.5 (2026-09-23)
+- [Fix] Stop `#flush` from running middleware twice on messages re-buffered after a failed flush (regression of #474 reintroduced by the failure/requeue path), which double-encrypted/compressed/serialized payloads and duplicated headers on retry.
+- [Fix] Re-check the producer liveness under `@buffer_mutex` in `#buffer`/`#buffer_many`, so a producer closed between the liveness check and the append raises `ProducerClosedError` instead of accepting the message into a closed producer's buffer and losing it silently.
+- [Fix] Restore both buffers when `#flush` fails before anything is dispatched (a middleware step raising, for example) instead of discarding the whole batch. Only `ProduceManyError` and `MessageInvalidError` were covered before.
+
 ## 2.10.4 (2026-08-26)
 - [Fix] Avoid a `FrozenError` on `Producer#close` when the producer is configured with a frozen string id (for example a frozen string literal, `config.id = "rspec"`).
 - [Fix] Emit `connection_pool.shutdown` only once when shutting down the global connection pool.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.10.4)
+    waterdrop (2.10.5)
       karafka-core (>= 2.5.12, < 3.0.0)
       karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
@@ -14,9 +14,9 @@ GEM
     connection_pool (3.0.2)
     drb (2.2.3)
     ffi (1.17.4)
-    io-console (0.8.2)
+    io-console (0.9.2)
     json (2.21.2)
-    karafka-core (2.6.2)
+    karafka-core (2.6.3)
       karafka-rdkafka (>= 0.20.0)
       logger (>= 1.6.0)
     karafka-rdkafka (0.28.0)
@@ -35,10 +35,10 @@ GEM
     ostruct (0.6.3)
     prism (1.9.0)
     rake (13.4.2)
-    reline (0.6.3)
+    reline (0.7.0)
       io-console (~> 0.5)
     ruby2_keywords (0.0.5)
-    simplecov (1.1.1)
+    simplecov (1.3.0)
     warning (1.6.0)
     zeitwerk (2.8.3)
 
@@ -61,9 +61,9 @@ CHECKSUMS
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
   ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
-  io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
+  io-console (0.9.2) sha256=efa74f891dd03c0939a931dfc6e74c2813d904763d456ea9762b0525e748db08
   json (2.21.2) sha256=1f1d3b7cf2b3ba1a69beca0bb6db13d5438b80bff3cd54cdaaa620b9b07c1c6a
-  karafka-core (2.6.2) sha256=c2fd7f277201b8ca97b824b364ad76bf776b8f5527bc422dafb71f7ae48d3a13
+  karafka-core (2.6.3) sha256=83a783895d8f69e5fd4b7e77617f6a43c129bbb905c93e1babb9ff299470ba4a
   karafka-rdkafka (0.28.0) sha256=fd845140c42341c5892215cac1e8adacda53b0461688f2e16379d82cb33eb006
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
@@ -72,11 +72,11 @@ CHECKSUMS
   ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
   rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
-  reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
+  reline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583d
   ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
-  simplecov (1.1.1) sha256=25825ef13f0b2e74694d769817dad6ab8e90131dabdaa666e522fea105521e78
+  simplecov (1.3.0) sha256=d9886307863c1ead47657dcbed869bfffd82318808b09014d8f0ff77dcfe7492
   warning (1.6.0) sha256=a49cdfae19fb77d19afff2efbe45f8ab759e9cd25b4e4ce2c79dbaf46bdb6c9e
-  waterdrop (2.10.4)
+  waterdrop (2.10.5)
   zeitwerk (2.8.3) sha256=2c85125a8467ce069e20123d1e709a08955c9d29c118c25b46b7b7fafdbb92e5
 
 BUNDLED WITH

data/README.md

@@ -1,7 +1,7 @@
 # WaterDrop
 
-[![Build Status](https://github.com/karafka/waterdrop/workflows/ci/badge.svg)](https://github.com/karafka/waterdrop/actions?query=workflow%3Aci)
-[![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
+[![Build Status](https://github.com/karafka/waterdrop/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/waterdrop/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](https://badge.fury.io/rb/waterdrop)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
 
 WaterDrop is a standalone gem that sends messages to Kafka easily with an extra validation layer. It is a part of the [Karafka](https://github.com/karafka/karafka) ecosystem.

data/bin/verify_kafka_warnings

@@ -4,6 +4,11 @@
 # Only specified warnings are allowed, all others should trigger failure
 
 allowed_patterns=(
+   # Enabling share groups (KIP-932) requires "share" in group.coordinator.rebalance.protocols
+   # on cp-kafka 8.3.0 (Kafka 4.x). That mechanism is deprecated in favour of the share.version
+   # feature flag (Kafka 5.0), but is still functionally required to enable share groups in this
+   # version, so the broker logs this deprecation warning on every startup. Expected here.
+   "group.coordinator.rebalance.protocols. is deprecated"
    "Performing controller activation"
    "registered with feature metadata.version"
    "TOPIC_ALREADY_EXISTS"

data/docker-compose.oauth.yml

@@ -49,6 +49,9 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       # Required by Confluent image when SASL is enabled
       # Also allow HTTP URLs for JWKS endpoint (Kafka 3.0+ blocks non-HTTPS by default)
       KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf -Dorg.apache.kafka.sasl.oauthbearer.allowed.urls=http://keycloak:8080/realms/kafka/protocol/openid-connect/certs"

data/docker-compose.sasl.yml

@@ -21,6 +21,9 @@ services:
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
       KAFKA_OPTS: ""

data/docker-compose.yml

@@ -21,5 +21,8 @@ services:
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: 'classic,consumer,share'
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 1
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer

data/lib/waterdrop/producer/async.rb

@@ -44,15 +44,18 @@ module WaterDrop
       #
       # @param messages [Array<Hash>] array with messages that comply with the
       #   {Contracts::Message} contract
+      # @param run_middleware [Boolean] should the middleware chain be run on the messages. Set to
+      #   false only when the messages have already been through middleware (for example when
+      #   re-dispatching a re-buffered flush batch) so it is not applied twice.
       #
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
       #
       # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
-      def produce_many_async(messages)
+      def produce_many_async(messages, run_middleware: true)
         dispatched = []
-        messages = middleware.run_many(messages)
+        messages = middleware.run_many(messages) if run_middleware
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/buffer.rb

@@ -15,12 +15,22 @@ module WaterDrop
         # The append runs under @buffer_mutex because flush/purge/close swap @messages for a fresh
         # array under the same lock. Without it, a concurrent swap between reading @messages and
         # appending would land the message in the orphaned old array and silently lose it.
+        #
+        # The liveness check is repeated under the same lock because `close` marks the producer
+        # `:closing` before its final flush acquires @buffer_mutex. Checking only above the lock
+        # would let a close complete in between, stranding the message in a closed producer's
+        # buffer while this call returned success.
         @monitor.instrument(
           "message.buffered",
           producer_id: id,
           message: message,
           buffer: @messages
-        ) { @buffer_mutex.synchronize { @messages << message } }
+        ) do
+          @buffer_mutex.synchronize do
+            ensure_active!
+            @messages << message
+          end
+        end
       end
 
       # Adds given messages into the internal producer buffer without flushing them to Kafka
@@ -34,14 +44,20 @@ module WaterDrop
 
         # The concat runs under @buffer_mutex for the same reason as #buffer: flush/purge/close swap
         # @messages under the lock, so an unguarded concat could append into an array that has just
-        # been captured for dispatch (or discarded), silently losing the messages.
+        # been captured for dispatch (or discarded), silently losing the messages. The liveness
+        # check is repeated under the lock for the same reason as in #buffer: a close that
+        # completes between the check above and the lock would otherwise strand the batch.
         @monitor.instrument(
           "messages.buffered",
           producer_id: id,
           messages: messages,
           buffer: @messages
         ) do
-          @buffer_mutex.synchronize { @messages.concat(messages) }
+          @buffer_mutex.synchronize do
+            ensure_active!
+            @messages.concat(messages)
+          end
+
           messages
         end
       end
@@ -53,7 +69,7 @@ module WaterDrop
         @monitor.instrument(
           "buffer.flushed_async",
           producer_id: id,
-          messages: @messages
+          messages: messages
         ) { flush(false) }
       end
 
@@ -64,7 +80,7 @@ module WaterDrop
         @monitor.instrument(
           "buffer.flushed_sync",
           producer_id: id,
-          messages: @messages
+          messages: messages
         ) { flush(true) }
       end
 
@@ -78,17 +94,47 @@ module WaterDrop
       # @note We use this method underneath to provide a different instrumentation for sync and
       #   async flushing within the public API
       def flush(sync)
-        data_for_dispatch = nil
+        fresh = nil
+        requeued = nil
+        data_for_dispatch = []
 
         @buffer_mutex.synchronize do
-          data_for_dispatch = @messages
+          requeued = @requeued
+          @requeued = []
+          fresh = @messages
           @messages = []
         end
 
+        # Middleware runs exactly once per message: on the fresh messages here, never again on the
+        # already-transformed ones coming back from a failed flush, which a second pass would
+        # corrupt.
+        #
+        # A middleware step raising here (a serializer, encryptor or schema lookup failing) would
+        # otherwise destroy both buffers, since they were emptied above and nothing has been
+        # dispatched yet to recover them from. Messages are run one by one so that on failure the
+        # ones whose chain completed are kept as returned and are not transformed again, while the
+        # failing one and those after it stay on the middleware path. The failing one is not
+        # requeued because that would dispatch it without the rest of its chain.
+        processed = []
+
+        begin
+          fresh.each { |message| processed << middleware.run(message) }
+        rescue
+          restore_undispatched(fresh.drop(processed.size), requeued.concat(processed))
+
+          raise
+        end
+
+        data_for_dispatch = requeued.concat(processed)
+
         # Do nothing if nothing to flush
         return data_for_dispatch if data_for_dispatch.empty?
 
-        sync ? produce_many_sync(data_for_dispatch) : produce_many_async(data_for_dispatch)
+        if sync
+          produce_many_sync(data_for_dispatch, run_middleware: false)
+        else
+          produce_many_async(data_for_dispatch, run_middleware: false)
+        end
       rescue Errors::ProduceManyError => e
         # A dispatch failed partway through the batch. Re-buffer the messages that never reached
         # librdkafka so a partial failure does not silently drop valid buffered messages. For a
@@ -106,15 +152,27 @@ module WaterDrop
         raise
       end
 
-      # Puts not-yet-dispatched messages back at the front of the buffer (preserving their original
-      # order relative to each other and to anything buffered concurrently), so a failed flush does
-      # not lose them.
+      # Puts both sets back after a failure that happened before anything reached librdkafka. They
+      # go to different buffers because middleware has run on one of them and not the other, which
+      # is what keeps it applied exactly once per message.
+      #
+      # @param fresh [Array<Hash>] messages whose middleware chain has not completed
+      # @param requeued [Array<Hash>] already middleware-processed messages
+      def restore_undispatched(fresh, requeued)
+        @buffer_mutex.synchronize do
+          @messages.unshift(*fresh)
+          @requeued.unshift(*requeued)
+        end
+      end
+
+      # Puts not-yet-dispatched messages back at the front of the retry buffer (preserving their
+      # original order relative to each other), so a failed flush does not lose them.
       #
-      # @param messages [Array<Hash>] messages to restore to the buffer
+      # @param messages [Array<Hash>] already middleware-processed messages to restore
       def requeue_unflushed(messages)
         return if messages.empty?
 
-        @buffer_mutex.synchronize { @messages.unshift(*messages) }
+        @buffer_mutex.synchronize { @requeued.unshift(*messages) }
       end
     end
   end

data/lib/waterdrop/producer/sync.rb

@@ -48,6 +48,9 @@ module WaterDrop
       #
       # @param messages [Array<Hash>] array with messages that comply with the
       #   {Contracts::Message} contract
+      # @param run_middleware [Boolean] should the middleware chain be run on the messages. Set to
+      #   false only when the messages have already been through middleware (for example when
+      #   re-dispatching a re-buffered flush batch) so it is not applied twice.
       #
       # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
       #   sent (can be used to verify offset, partition, etc via `#create_result`)
@@ -63,8 +66,8 @@ module WaterDrop
       #   errors), returning handles guarantees the same type in the `dispatched` collection,
       #   allowing uniform error handling. Each handle is in its final state after this method
       #   returns, so you can call `handle.create_result` to obtain the delivery report if needed.
-      def produce_many_sync(messages)
-        messages = middleware.run_many(messages)
+      def produce_many_sync(messages, run_middleware: true)
+        messages = middleware.run_many(messages) if run_middleware
         messages.each { |message| validate_message!(message) }
 
         dispatched = []

data/lib/waterdrop/producer.rb

@@ -39,8 +39,6 @@ module WaterDrop
     attr_reader :id
     # @return [Status] producer status object
     attr_reader :status
-    # @return [Array] internal messages buffer
-    attr_reader :messages
     # @return [Object] monitor we want to use
     attr_reader :monitor
     # @return [Object] dry-configurable config object
@@ -71,6 +69,9 @@ module WaterDrop
 
       @status = Status.new
       @messages = []
+      # Messages re-buffered after a failed flush. Kept apart from @messages because they have
+      # already passed through middleware and must not be transformed again on the next flush.
+      @requeued = []
 
       # Instrument producer creation for global listeners
       class_monitor.instrument(
@@ -177,6 +178,12 @@ module WaterDrop
       @client
     end
 
+    # @return [Array] internal messages buffer awaiting a flush. Includes both freshly buffered
+    #   messages and any messages re-buffered after a failed flush that are waiting to be retried.
+    def messages
+      @requeued + @messages
+    end
+
     # Returns the number of messages in the librdkafka producer queue.
     #
     # This count includes:
@@ -232,6 +239,7 @@ module WaterDrop
       @monitor.instrument("buffer.purged", producer_id: id) do
         @buffer_mutex.synchronize do
           @messages = []
+          @requeued = []
         end
 
         # We should not purge if there is no client initialized
@@ -321,6 +329,7 @@ module WaterDrop
             return false unless @client
             return false unless @status.connected?
             return false unless @messages.empty?
+            return false unless @requeued.empty?
             return false unless @operations_in_progress.value.zero?
 
             @status.disconnecting!
@@ -354,6 +363,7 @@ module WaterDrop
       return false unless @client
       return false unless @status.connected?
       return false unless @messages.empty?
+      return false unless @requeued.empty?
       return false if @transaction_mutex.locked?
       return false if @operating_mutex.locked?
 
@@ -503,7 +513,7 @@ module WaterDrop
       # Try to get buffer info safely
       if @buffer_mutex.try_lock
         begin
-          parts << "buffer_size=#{@messages.size}"
+          parts << "buffer_size=#{@messages.size + @requeued.size}"
         ensure
           @buffer_mutex.unlock
         end

data/lib/waterdrop/version.rb

@@ -2,5 +2,5 @@
 
 module WaterDrop
   # Current WaterDrop version
-  VERSION = "2.10.4"
+  VERSION = "2.10.5"
 end

data/package-lock.json

@@ -286,9 +286,9 @@
       }
     },
     "node_modules/smol-toml": {
-      "version": "1.7.1",
-      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
-      "integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==",
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
+      "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
       "dev": true,
       "license": "BSD-3-Clause",
       "engines": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.10.4
+  version: 2.10.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -160,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Kafka messaging made easy!
 test_files: []