waterdrop 2.9.0 → 2.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38a7f384d48b06062104341df54cf2532c6e338549dd370507d8b2e820721c73
-  data.tar.gz: 2b66e9f62a94a4fb1baafc8492f9bcb9d51cee3bebc9de937da2860576f82bfb
+  metadata.gz: 5a840a99425c1700eb3ea2cad5da08279e11ba0f4a2600b046dcef14bca9255b
+  data.tar.gz: 6107f58c3ed66912e56379660a021eb87c1e18a0d7fee7702460c1e90b75fea3
 SHA512:
-  metadata.gz: 941854a59acf37d5f19d49186cd0950ca699a441653995a1f09cec740fd44c98880364f954aada62377a2313a61e1e164d20e38601384a9f77c3671156150d5b
-  data.tar.gz: 70fdbebdff760f2fba63fc001cfe4c151a25c64b9e9ce61f5fbc4f17b9c13b392f06a04cff41d02ae32edd015d2ca801b9fd744f7804417aff2a16bde6d4a75e
+  metadata.gz: 454ff01bc3baa3c2b47c46c6538dd8310c3b2af10cafe3551f00630e78d5980b7afeec7eb2047f769b24d0d6106aba4366e04e99d91045154a4ddb0b34edd4b1
+  data.tar.gz: '0980ac5585f18983d4d6918f99cf8bf070160d5c99964459c714b532195d74372785dd9b3e40366fae709f92aea9c364e2b71d33c3ffadde0528f60e863d8348'

data/.gitignore

@@ -67,3 +67,6 @@ pickle-email-*.html
 .yardoc
 
 .byebug_history
+
+# Integration test lock files (generated locally with paths, CI runs bundle install)
+test/integrations/**/Gemfile.lock

data/.ruby-version

@@ -1 +1 @@
-4.0.2
+4.0.3

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # WaterDrop changelog
 
+## 2.10.0 (2026-05-07)
+- [Fix] Clean up native rdkafka client, global instrumentation callbacks, and poller registration when `init_transactions` fails during producer client construction. Previously, each failed attempt permanently leaked native threads, pipe file descriptors, and callback registry entries because the started `rd_kafka_t` handle was abandoned without being destroyed.
+- **[Breaking]** Skip emitting librdkafka statistics when nothing is subscribed to `statistics.emitted` at the time the underlying rdkafka client is constructed. When no listener is present at build time, `statistics.interval.ms` is forced to `0` regardless of user configuration and the statistics callback is not registered, saving substantial allocations in the hot path (no JSON parsing, no statistics hash materialization, no decoration work). To use statistics, subscribe a listener to `statistics.emitted` BEFORE the first producer use (before the underlying client is lazily initialized).
+- **[Breaking]** Raise `WaterDrop::Errors::StatisticsNotEnabledError` when attempting to subscribe to `statistics.emitted` (either via block or via a listener that responds to `on_statistics_emitted`) on a monitor where librdkafka statistics have been disabled at client build time. This replaces the "silent nothing" failure mode with an immediate, actionable error that pinpoints the timing mistake.
+- [Feature] Add tombstone API (`#tombstone_sync`, `#tombstone_async`, `#tombstone_many_sync`, `#tombstone_many_async`) for producing tombstone records (nil-payload messages) with required key and partition validation. Works with variants.
+- [Fix] Add `ensure_same_process!` to `Poller#unregister` for fork safety. Without this, a child process that inherited a pre-fork producer would deadlock on `producer.close` because `unregister` waited on a latch the dead parent poller thread would never release.
+
 ## 2.9.0 (2026-04-08)
 - [Fix] Use `delete` in the variant ensure block to avoid leaving stale nil entries in `Fiber.current.waterdrop_clients` and prevent memory leaks in long-running processes (#836).
 - [Fix] Exclude test files, `.github/`, and `log/` directories from gem releases to reduce package size.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.9.0)
+    waterdrop (2.10.0)
       karafka-core (>= 2.5.12, < 3.0.0)
       karafka-rdkafka (>= 0.24.0)
       zeitwerk (~> 2.3)
@@ -28,7 +28,7 @@ GEM
       rake (> 12)
     logger (1.7.0)
     mini_portile2 (2.8.9)
-    minitest (6.0.2)
+    minitest (6.0.6)
       drb (~> 2.0)
       prism (~> 1.5)
     mocha (3.1.0)
@@ -74,7 +74,7 @@ CHECKSUMS
   karafka-rdkafka (0.25.0) sha256=67b316b942cf9ff7e9d7bbf9029e6f2d91eba97b4c9dc93b9f49fd207dfb80f8
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
-  minitest (6.0.2) sha256=db6e57956f6ecc6134683b4c87467d6dd792323c7f0eea7b93f66bd284adbc3d
+  minitest (6.0.6) sha256=153ea36d1d987a62942382b61075745042a2b3123b1cd48f4c3675af9cc7d6f1
   mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56
   ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
@@ -85,7 +85,7 @@ CHECKSUMS
   simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
   simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
   warning (1.5.0) sha256=0f12c49fea0c06757778eefdcc7771e4fd99308901e3d55c504d87afdd718c53
-  waterdrop (2.9.0)
+  waterdrop (2.10.0)
   zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd
 
 BUNDLED WITH

data/bin/verify_kafka_warnings

@@ -6,6 +6,7 @@
 allowed_patterns=(
    "Performing controller activation"
    "registered with feature metadata.version"
+   "TOPIC_ALREADY_EXISTS"
 )
 
 # Get all warnings

data/config/locales/errors.yml

@@ -61,6 +61,11 @@ en:
       headers_invalid_key_type: all headers keys need to be of type String
       headers_invalid_value_type: all headers values need to be strings or arrays of strings
 
+    tombstone:
+      missing: must be present
+      key_format: must be a non-empty string
+      partition_format: must be an integer greater or equal to 0
+
     transactional_offset:
       consumer_format: 'must respond to #consumer_group_metadata_pointer method'
       message_format: 'must respond to #topic, #partition and #offset'

data/docker-compose.sasl.yml

@@ -0,0 +1,26 @@
+services:
+  kafka-sasl:
+    image: confluentinc/cp-kafka:8.2.0
+    container_name: kafka-sasl
+    ports:
+      - "9095:9095"
+    environment:
+      CLUSTER_ID: kafka-sasl-cluster-1
+      KAFKA_BROKER_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-sasl:9093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,SASL_PLAINTEXT://:9095
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-sasl:9092,SASL_PLAINTEXT://127.0.0.1:9095
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAINTEXT
+      KAFKA_LISTENER_NAME_SASL__PLAINTEXT_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="admin-secret" user_admin="admin-secret" user_testuser="testuser-secret";'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
+      KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
+      KAFKA_OPTS: ""

data/lib/waterdrop/clients/rdkafka.rb

@@ -11,68 +11,185 @@ module WaterDrop
         # @param producer [WaterDrop::Producer] producer instance with its config, etc
         # @note We overwrite this that way, because we do not care
         def new(producer)
-          kafka_config = producer.config.kafka.to_h
           monitor = producer.config.monitor
+          kafka_config, statistics_enabled = prepare_statistics(
+            producer.config.kafka.to_h,
+            monitor
+          )
 
-          # When FD polling is enabled, we disable the native librdkafka polling thread
-          # and use our own Ruby-based poller instead
-          producer_options = { native_kafka_auto_start: false }
+          client = build_rdkafka_client(producer, kafka_config)
+
+          register_instrumentation_callbacks(
+            producer,
+            client,
+            monitor,
+            statistics_enabled: statistics_enabled
+          )
+
+          # This callback is not global and is per client, thus we do not have to wrap it with a
+          # callbacks manager to make it work
+          client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
+            producer.id,
+            producer.transactional?,
+            monitor
+          )
+
+          subscribe_oauth_listener(producer, monitor)
+          activate_client(producer, client, kafka_config)
+
+          client
+        end
+
+        private
+
+        # Decides whether librdkafka statistics should be enabled for this client and returns
+        # the (possibly mutated) kafka config together with the decision.
+        #
+        # When no one is subscribed to `statistics.emitted` at the time the underlying rdkafka
+        # client is being built, we force `statistics.interval.ms` to 0 regardless of user
+        # configuration. This prevents librdkafka from computing statistics periodically and
+        # saves a significant number of allocations on the Ruby side (no JSON parsing, no
+        # statistics hash materialization, no decorator work). Any listener subscribed after
+        # the client has been built will not receive `statistics.emitted` events because
+        # librdkafka never emits them in the first place — to use statistics, subscribe a
+        # listener BEFORE the first producer use.
+        #
+        # When statistics end up disabled (either because the user explicitly set the interval
+        # to 0, or because we forced it to 0 here), we freeze the statistics listener slot on
+        # the monitor. Any later subscription attempt raises instead of silently being a no-op,
+        # surfacing the timing mistake to the user immediately.
+        #
+        # @param kafka_config [Hash] kafka config hash taken from the producer config
+        # @param monitor [WaterDrop::Instrumentation::Monitor] per-producer monitor
+        # @return [Array] two-element array `[kafka_config, statistics_enabled]`. The returned
+        #   hash is a duped copy when we need to mutate the interval, so the producer's own
+        #   config hash is never touched.
+        def prepare_statistics(kafka_config, monitor)
+          statistics_enabled = kafka_config[:"statistics.interval.ms"].to_i.positive?
+
+          if statistics_enabled && !statistics_listener?(monitor)
+            kafka_config = kafka_config.dup
+            kafka_config[:"statistics.interval.ms"] = 0
+            statistics_enabled = false
+          end
+
+          monitor.freeze_statistics_listeners! unless statistics_enabled
 
+          [kafka_config, statistics_enabled]
+        end
+
+        # Instantiates the underlying rdkafka producer with the correct polling options. When
+        # FD polling is enabled, we disable librdkafka's native background polling thread and
+        # use our own Ruby-based poller instead.
+        #
+        # @param producer [WaterDrop::Producer]
+        # @param kafka_config [Hash]
+        # @return [::Rdkafka::Producer]
+        def build_rdkafka_client(producer, kafka_config)
+          producer_options = { native_kafka_auto_start: false }
           producer_options[:run_polling_thread] = false if producer.fd_polling?
 
-          client = ::Rdkafka::Config.new(kafka_config).producer(**producer_options)
+          ::Rdkafka::Config.new(kafka_config).producer(**producer_options)
+        end
 
-          # Register statistics runner for this particular type of callbacks
-          ::Karafka::Core::Instrumentation.statistics_callbacks.add(
-            producer.id,
-            Instrumentation::Callbacks::Statistics.new(
+        # Registers the global callbacks (statistics, error, oauth refresh) for this producer
+        # on the shared `Karafka::Core::Instrumentation` managers. The statistics callback is
+        # only registered when librdkafka is actually going to emit statistics — otherwise it
+        # would never fire and would only waste memory and a manager slot.
+        #
+        # @param producer [WaterDrop::Producer]
+        # @param client [::Rdkafka::Producer]
+        # @param monitor [WaterDrop::Instrumentation::Monitor]
+        # @param statistics_enabled [Boolean]
+        def register_instrumentation_callbacks(producer, client, monitor, statistics_enabled:)
+          if statistics_enabled
+            ::Karafka::Core::Instrumentation.statistics_callbacks.add(
               producer.id,
-              client.name,
-              monitor,
-              producer.config.statistics_decorator
+              Instrumentation::Callbacks::Statistics.new(
+                producer.id,
+                client.name,
+                monitor,
+                producer.config.statistics_decorator
+              )
             )
-          )
+          end
 
-          # Register error tracking callback
           ::Karafka::Core::Instrumentation.error_callbacks.add(
             producer.id,
             Instrumentation::Callbacks::Error.new(producer.id, client.name, monitor)
           )
 
-          # Register oauth bearer refresh for this particular type of callbacks
           ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
             producer.id,
             Instrumentation::Callbacks::OauthbearerTokenRefresh.new(client, monitor)
           )
+        end
 
-          # This callback is not global and is per client, thus we do not have to wrap it with a
-          # callbacks manager to make it work
-          client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
-            producer.id,
-            producer.transactional?,
-            monitor
-          )
-
+        # Subscribes the oauth bearer token refresh listener to the monitor if one is configured.
+        #
+        # We need to subscribe it here because we want it to be ready before any producer
+        # callbacks run. In theory because the WaterDrop rdkafka producer is lazy loaded, the
+        # user would have enough time to subscribe it himself, but then it would not coop with
+        # auto-configuration coming from Karafka. The way it is done here, if it is configured
+        # it will be subscribed and if not, the user always can subscribe it himself as long as
+        # it is done prior to first usage.
+        #
+        # @param producer [WaterDrop::Producer]
+        # @param monitor [WaterDrop::Instrumentation::Monitor]
+        def subscribe_oauth_listener(producer, monitor)
           oauth_listener = producer.config.oauth.token_provider_listener
-          # We need to subscribe the oauth listener here because we want it to be ready before
-          # any producer callbacks run. In theory because WaterDrop rdkafka producer is lazy loaded
-          # we would have enough time to make user subscribe it himself, but then it would not
-          # coop with auto-configuration coming from Karafka. The way it is done below, if it is
-          # configured it will be subscribed and if not, user always can subscribe it himself as
-          # long as it is done prior to first usage
           monitor.subscribe(oauth_listener) if oauth_listener
+        end
 
+        # Transitions the freshly built client into an active state: starts the native side,
+        # registers it with our FD poller (when FD polling is enabled), and initializes
+        # transactions if the user configured a transactional id. Must run last so all
+        # callbacks are already wired up before the client goes live.
+        #
+        # If any step after `client.start` fails (most commonly `init_transactions` timing
+        # out when Kafka is unreachable), we must clean up everything that was already set up:
+        # unregister from the poller, remove the global instrumentation callbacks, and close
+        # the native client. Without this, each failed attempt leaks native threads, pipe
+        # file descriptors, and callback registry entries permanently.
+        #
+        # @param producer [WaterDrop::Producer]
+        # @param client [::Rdkafka::Producer]
+        # @param kafka_config [Hash]
+        def activate_client(producer, client, kafka_config)
           client.start
 
-          # Register with poller if FD polling is enabled
-          # Uses the producer's configured poller (custom or global singleton)
-          # This must happen after client.start to ensure the client is ready
+          # Register with poller if FD polling is enabled. Uses the producer's configured poller
+          # (custom or global singleton). This must happen after client.start to ensure the
+          # client is ready.
           producer.poller.register(producer, client) if producer.fd_polling?
 
-          # Switch to the transactional mode if user provided the transactional id
+          # Switch to transactional mode if user provided a transactional id
           client.init_transactions if kafka_config.key?(:"transactional.id")
+        rescue
+          # Unwind everything we set up before re-raising:
+          # 1. Unregister from poller (if we registered)
+          producer.poller.unregister(producer) if producer.fd_polling?
 
-          client
+          # 2. Remove global instrumentation callbacks so they don't accumulate
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(producer.id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(producer.id)
+          ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(producer.id)
+
+          # 3. Close the native client to join its threads and release pipe FDs
+          client.close
+
+          raise
+        end
+
+        # Checks whether there is at least one subscriber to the `statistics.emitted` event on
+        # the per-producer monitor. We use this at client build time to decide whether to enable
+        # librdkafka statistics at all.
+        #
+        # @param monitor [WaterDrop::Instrumentation::Monitor] per-producer monitor
+        # @return [Boolean] true if any listener is registered for `statistics.emitted`
+        def statistics_listener?(monitor)
+          listeners = monitor.listeners["statistics.emitted"]
+          listeners && !listeners.empty?
         end
       end
     end

data/lib/waterdrop/contracts/tombstone.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract for validating tombstone-specific message requirements.
+    # Tombstones require a non-nil key and an explicit partition.
+    #
+    # @note Topic, headers, and other standard message attributes are validated separately
+    #   by the {Message} contract during the produce delegation flow.
+    class Tombstone < ::Karafka::Core::Contractable::Contract
+      configure do |config|
+        config.error_messages = YAML.safe_load_file(
+          File.join(WaterDrop.gem_root, "config", "locales", "errors.yml")
+        ).fetch("en").fetch("validations").fetch("tombstone")
+      end
+
+      required(:key) { |val| val.is_a?(String) && !val.empty? }
+      required(:partition) { |val| val.is_a?(Integer) && val >= 0 }
+    end
+  end
+end

data/lib/waterdrop/errors.rb

@@ -59,6 +59,13 @@ module WaterDrop
     # Raised when an error occurs in the polling loop
     PollerError = Class.new(BaseError)
 
+    # Raised when trying to subscribe to `statistics.emitted` after the underlying rdkafka client
+    # has been built without any listener present at build time. In that case, librdkafka
+    # statistics are disabled entirely for performance, and late subscriptions would silently
+    # receive nothing. To fix: subscribe the listener BEFORE first producer use (i.e. before the
+    # underlying client is lazily initialized).
+    StatisticsNotEnabledError = Class.new(BaseError)
+
     # Raised when during messages producing something bad happened inline
     class ProduceManyError < ProduceError
       attr_reader :dispatched

data/lib/waterdrop/instrumentation/monitor.rb

@@ -6,6 +6,15 @@ module WaterDrop
     # By default uses our internal notifications bus but can be used with
     # `ActiveSupport::Notifications` as well
     class Monitor < ::Karafka::Core::Monitoring::Monitor
+      # Event name for librdkafka statistics emissions
+      STATISTICS_EVENT = "statistics.emitted"
+
+      # Method name a listener object must implement in order to receive
+      # `statistics.emitted` events via object-based subscription
+      STATISTICS_LISTENER_METHOD = :on_statistics_emitted
+
+      private_constant :STATISTICS_EVENT, :STATISTICS_LISTENER_METHOD
+
       # @param notifications_bus [Object] either our internal notifications bus or
       #   `ActiveSupport::Notifications`
       # @param namespace [String, nil] namespace for events or nil if no namespace
@@ -14,6 +23,58 @@ module WaterDrop
         namespace = nil
       )
         super
+        @statistics_listeners_frozen = false
+      end
+
+      # Marks this monitor as no longer accepting new subscriptions to `statistics.emitted`.
+      # Called by the rdkafka client builder when it decides to leave librdkafka statistics
+      # disabled (because no listener was present at build time). Any subsequent attempt to
+      # subscribe to `statistics.emitted` — either via a block or via a listener object that
+      # responds to `on_statistics_emitted` — will raise
+      # `WaterDrop::Errors::StatisticsNotEnabledError` instead of silently doing nothing.
+      def freeze_statistics_listeners!
+        @statistics_listeners_frozen = true
+      end
+
+      # Subscribes to the notifications bus, raising if the user tries to subscribe to
+      # `statistics.emitted` after statistics have been disabled at client build time. This
+      # prevents the "silent nothing" pitfall where a user expects statistics but no events
+      # ever arrive because librdkafka statistics were turned off entirely.
+      #
+      # @param event_id_or_listener [String, Symbol, Object] event id (with block) or listener
+      # @param block [Proc, nil] handler block when subscribing to a named event
+      # @raise [WaterDrop::Errors::StatisticsNotEnabledError] when the subscription targets
+      #   `statistics.emitted` and this monitor has been frozen for statistics
+      def subscribe(event_id_or_listener, &block)
+        if @statistics_listeners_frozen && targets_statistics?(event_id_or_listener, block)
+          raise Errors::StatisticsNotEnabledError, <<~MSG.tr("\n", " ").strip
+            Cannot subscribe to `statistics.emitted` after the producer has been connected.
+            Statistics are disabled because no listener was subscribed before the underlying
+            rdkafka client was built, so librdkafka is not emitting statistics at all.
+            Subscribe your listener BEFORE the first producer use (before the underlying
+            client is lazily initialized), or explicitly keep statistics enabled by leaving
+            a listener in place at build time.
+          MSG
+        end
+
+        super
+      end
+
+      private
+
+      # Determines whether a subscription call targets `statistics.emitted`. Handles both
+      # block-based subscription (where the first argument is the event id string) and
+      # listener-object subscription (where the listener responds to `on_statistics_emitted`).
+      #
+      # @param event_id_or_listener [String, Symbol, Object]
+      # @param block [Proc, nil]
+      # @return [Boolean]
+      def targets_statistics?(event_id_or_listener, block)
+        if block
+          event_id_or_listener.to_s == STATISTICS_EVENT
+        else
+          event_id_or_listener.respond_to?(STATISTICS_LISTENER_METHOD)
+        end
       end
     end
   end

data/lib/waterdrop/polling/latch.rb

@@ -18,6 +18,7 @@ module WaterDrop
     # This ensures the producer is fully drained and removed from the poller
     # before returning control to the caller, preventing race conditions.
     class Latch
+      # Initializes a new latch in the unreleased state.
       def initialize
         @mutex = Mutex.new
         @cv = ConditionVariable.new

data/lib/waterdrop/polling/poller.rb

@@ -47,6 +47,8 @@ module WaterDrop
       # @return [Integer] unique identifier for this poller instance
       attr_reader :id
 
+      # Initializes an empty poller with no registered producers. The background thread is
+      # not started until the first producer is registered.
       def initialize
         @id = self.class.next_id
         @mutex = Mutex.new
@@ -142,6 +144,8 @@ module WaterDrop
       # This matches the threaded polling behavior which drains without timeout
       # @param producer [WaterDrop::Producer] the producer instance
       def unregister(producer)
+        ensure_same_process!
+
         state, thread = @mutex.synchronize { [@producers[producer.id], @thread] }
 
         return unless state

data/lib/waterdrop/producer/tombstone.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for tombstone producer operations
+    #
+    # Tombstone records are Kafka messages with a nil payload, used to signal deletion of a key
+    # in compacted topics. This module provides a dedicated API so users don't have to manually
+    # construct `produce_*(topic:, key:, payload: nil, ...)` calls.
+    module Tombstone
+      # Produces a tombstone message to Kafka and waits for it to be delivered
+      #
+      # @param message [Hash] hash with at least `:topic`, `:key`, and `:partition` keys.
+      #   `:payload` is not accepted — it will be silently removed if present.
+      #
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
+      #
+      # @raise [Errors::MessageInvalidError] When `:key` or `:partition` is missing
+      def tombstone_sync(message)
+        produce_sync(prepare_tombstone(message))
+      end
+
+      # Produces a tombstone message to Kafka and does not wait for results
+      #
+      # @param message [Hash] hash with at least `:topic`, `:key`, and `:partition` keys.
+      #   `:payload` is not accepted — it will be silently removed if present.
+      #
+      # @return [Rdkafka::Producer::DeliveryHandle] delivery handle
+      #
+      # @raise [Errors::MessageInvalidError] When `:key` or `:partition` is missing
+      def tombstone_async(message)
+        produce_async(prepare_tombstone(message))
+      end
+
+      # Produces many tombstone messages to Kafka and waits for them to be delivered
+      #
+      # @param messages [Array<Hash>] array of hashes, each with `:topic`, `:key`, and
+      #   `:partition` keys
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles
+      #
+      # @raise [Errors::MessageInvalidError] When any message is missing `:key` or `:partition`
+      def tombstone_many_sync(messages)
+        produce_many_sync(messages.map { |message| prepare_tombstone(message) })
+      end
+
+      # Produces many tombstone messages to Kafka and does not wait for them to be delivered
+      #
+      # @param messages [Array<Hash>] array of hashes, each with `:topic`, `:key`, and
+      #   `:partition` keys
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles
+      #
+      # @raise [Errors::MessageInvalidError] When any message is missing `:key` or `:partition`
+      def tombstone_many_async(messages)
+        produce_many_async(messages.map { |message| prepare_tombstone(message) })
+      end
+
+      private
+
+      # Validates and prepares a tombstone message by ensuring required keys are present
+      # and setting payload to nil
+      #
+      # @param message [Hash] the original message hash
+      # @return [Hash] a new message hash with payload set to nil
+      # @raise [Errors::MessageInvalidError] when key or partition is missing
+      def prepare_tombstone(message)
+        message = message.dup
+        message.delete(:payload)
+        message[:payload] = nil
+
+        Contracts::Tombstone.new.validate!(message, Errors::MessageInvalidError)
+
+        message
+      end
+    end
+  end
+end

data/lib/waterdrop/producer/variant.rb

@@ -71,6 +71,7 @@ module WaterDrop
         Async,
         Buffer,
         Sync,
+        Tombstone,
         Transactions
       ].each do |scope|
         scope.instance_methods(false).each do |method_name|

data/lib/waterdrop/producer.rb

@@ -7,6 +7,7 @@ module WaterDrop
     include Sync
     include Async
     include Buffer
+    include Tombstone
     include Transactions
     include Idempotence
     include ClassMonitor

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = "2.9.0"
+  VERSION = "2.10.0"
 end

data/renovate.json

@@ -45,5 +45,8 @@
   "minimumReleaseAge": "7 days",
   "labels": [
     "dependencies"
-  ]
+  ],
+  "lockFileMaintenance": {
+    "enabled": true
+  }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.9.0
+  version: 2.10.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -82,6 +82,7 @@ files:
 - bin/verify_topics_naming
 - config/locales/errors.yml
 - docker-compose.oauth.yml
+- docker-compose.sasl.yml
 - docker-compose.yml
 - lib/waterdrop.rb
 - lib/waterdrop/clients/buffered.rb
@@ -93,6 +94,7 @@ files:
 - lib/waterdrop/contracts/config.rb
 - lib/waterdrop/contracts/message.rb
 - lib/waterdrop/contracts/poller_config.rb
+- lib/waterdrop/contracts/tombstone.rb
 - lib/waterdrop/contracts/transactional_offset.rb
 - lib/waterdrop/contracts/variant.rb
 - lib/waterdrop/errors.rb
@@ -125,6 +127,7 @@ files:
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/producer/testing.rb
+- lib/waterdrop/producer/tombstone.rb
 - lib/waterdrop/producer/transactions.rb
 - lib/waterdrop/producer/variant.rb
 - lib/waterdrop/version.rb