waterdrop 2.6.13 → 2.7.0.alpha1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '06984e8584d8b1ed3e6f854be6ca1cd1957acab63d6640c41087a42572da670a'
-  data.tar.gz: 9b0c94f047b5e21a554ecad5f32639a761aa32fbab9f5776a2470fdfe4adb675
+  metadata.gz: 938018b39bca57a68925d61e00a1347c34d3ad536fb73c910589065fc8e6cbca
+  data.tar.gz: cdad580ebf6e91d51464fb2336f9c78896f3d1b03e96919463c99f141b5dc5ff
 SHA512:
-  metadata.gz: 610ec69f42c6d209e3d024825d62ecd25e7d50ad212d14b4b2db7534463f63e5cee4662b0412db559d1537fa28ef76acd3b22418f686fcea6c6406a1cb6cec54
-  data.tar.gz: af35df96c8566d1fef3d16898553277c3d15a572ab8fc563858048beae70915a761109a32cd101b22123174f16dc601f7f28c2699fae20126d5dd9dc519fcd23
+  metadata.gz: 15e328bfe3135a14b3f61019185bcbb88c56ed2ed01e2a6523439229ee709940c0f38e4d2259e208c785fb06c2326fb290423c88d5213f730120b0a11ea8a152
+  data.tar.gz: f6642bb5dbc1cb84e923bc5bdc43417bf3f88215eab4a537e3fa14290726e935a45aec0cf640962f3e48615310d639ed1f592cf324b9963e1ff1dc0dabb8cbe2

data/.github/workflows/ci.yml

@@ -22,7 +22,6 @@ jobs:
           - '3.2'
           - '3.1'
           - '3.0'
-          - '2.7'
         include:
           - ruby: '3.3'
             coverage: 'true'
@@ -49,25 +48,15 @@ jobs:
 
       - name: Install latest bundler
         run: |
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            gem install bundler -v 2.4.22 --no-document
-            gem update --system 3.4.22 --no-document
-          else
-            gem install bundler --no-document
-            gem update --system --no-document
-          fi
+          gem install bundler --no-document
+          gem update --system --no-document
 
           bundle config set without 'tools benchmarks docs'
 
       - name: Bundle install
         run: |
           bundle config set without development
-
-          if [[ "$(ruby -v | awk '{print $2}')" == 2.7.8* ]]; then
-            BUNDLER_VERSION=2.4.22 bundle install --jobs 4 --retry 3
-          else
-            bundle install --jobs 4 --retry 3
-          fi
+          bundle install --jobs 4 --retry 3
 
       - name: Run all tests
         env:

data/CHANGELOG.md

@@ -1,5 +1,87 @@
 # WaterDrop changelog
 
+## 2.7.0 (Unreleased)
+
+This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
+
+- **[Breaking]** Drop Ruby `2.7` support.
+- **[Breaking]** Change default timeouts so final delivery `message.timeout.ms` is less that `max_wait_time` so we do not end up with not final verdict.
+- **[Breaking]** Update all the time related configuration settings to be in `ms` and not mixed.
+- [Enhancement] Introduce `instrument_on_wait_queue_full` flag (defaults to `true`) to be able to configure whether non critical (retryable) queue full errors should be instrumented in the error pipeline. Useful when building high-performance pipes with WaterDrop queue retry backoff as a throttler.
+
+### Upgrade Notes
+
+**PLEASE MAKE SURE TO READ AND APPLY THEM!**
+
+#### Time Settings Format Alignment
+
+**All** time-related values are now configured in milliseconds instead of some being in seconds and some in milliseconds.
+
+The values that were changed from seconds to milliseconds are:
+
+- `max_wait_timeout`
+- `wait_timeout`
+- `wait_backoff_on_queue_full`
+- `wait_timeout_on_queue_full`
+- `wait_backoff_on_transaction_command, default`
+
+If you have configured any of those yourself, please replace the seconds representation with milliseconds:
+
+```ruby
+producer = WaterDrop::Producer.new
+
+producer.setup do |config|
+  config.deliver = true
+
+  # Replace this:
+  config.wait_timeout = 30
+
+  # With
+  config.wait_timeout = 30_000
+  # ...
+end
+```
+
+#### Defaults Alignment
+
+In this release, we've updated our default settings to address a crucial issue: previous defaults could lead to inconclusive outcomes in synchronous operations due to wait timeout errors. Users often mistakenly believed that a message dispatch was halted because of these errors when, in fact, the timeout was related to awaiting the final dispatch verdict, not the dispatch action itself.
+
+The new defaults in WaterDrop 2.7.0 eliminate this confusion by ensuring synchronous operation results are always transparent and conclusive. This change aims to provide a straightforward understanding of wait timeout errors, reinforcing that they reflect the wait state, not the dispatch success.
+
+Below, you can find a table with what has changed, the new defaults, and the current ones in case you want to retain the previous behavior:
+
+<table>
+  <thead>
+    <tr>
+      <th>Config</th>
+      <th>Previous Default</th>
+      <th>New Default</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>root <code>max_wait_timeout</code></td>
+      <td>5000 ms (5 seconds)</td>
+      <td>60000 ms (60 seconds)</td>
+    </tr>
+    <tr>
+      <td>kafka <code>message.timeout.ms</code></td>
+      <td>300000 ms (5 minutes)</td>
+      <td>55000 ms (55 seconds)</td>
+    </tr>
+    <tr>
+      <td>kafka <code>transaction.timeout.ms</code></td>
+      <td>60000 ms (1 minute)</td>
+      <td>45000 ms (45 seconds)</td>
+    </tr>
+  </tbody>
+</table>
+
+This alignment ensures that when using sync operations or invoking `#wait`, any exception you get should give you a conclusive and final delivery verdict.
+
+## 2.6.14 (2024-02-06)
+- [Enhancement] Instrument `producer.connected` and `producer.closing` lifecycle events.
+
 ## 2.6.13 (2024-01-29)
 - [Enhancement] Expose `#partition_count` for building custom partitioners that need to be aware of number of partitions on a given topic.
 

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.6.13)
-      karafka-core (>= 2.2.3, < 3.0.0)
+    waterdrop (2.7.0.alpha1)
+      karafka-core (>= 2.4.0.alpha1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3)
+    activesupport (7.1.3.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -23,40 +23,37 @@ GEM
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
-    diff-lcs (1.5.0)
+    diff-lcs (1.5.1)
     docile (1.4.0)
-    drb (2.2.0)
-      ruby2_keywords
-    factory_bot (6.4.5)
+    drb (2.2.1)
+    factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     ffi (1.16.3)
-    i18n (1.14.1)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.2.7)
-      concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.9, < 0.15.0)
-    karafka-rdkafka (0.14.7)
+    karafka-core (2.4.0.alpha1)
+      karafka-rdkafka (>= 0.15.0.alpha1, < 0.16.0)
+    karafka-rdkafka (0.15.0.alpha1)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     mini_portile2 (2.8.5)
-    minitest (5.21.2)
+    minitest (5.22.2)
     mutex_m (0.2.0)
     rake (13.1.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.0)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
-    ruby2_keywords (0.0.5)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -65,10 +62,10 @@ GEM
     simplecov_json_formatter (0.1.4)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
-  ruby
+  arm64-darwin-22
   x86_64-linux
 
 DEPENDENCIES

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.3
+    image: confluentinc/cp-kafka:7.6.0
 
     ports:
       - 9092:9092

data/lib/waterdrop/config.rb

@@ -12,7 +12,12 @@ module WaterDrop
       'client.id': 'waterdrop',
       # emit librdkafka statistics every five seconds. This is used in instrumentation.
       # When disabled, part of metrics will not be published and available.
-      'statistics.interval.ms': 5_000
+      'statistics.interval.ms': 5_000,
+      # We set it to a value that is lower than `max_wait_time` to have a final verdict upon sync
+      # delivery
+      'message.timeout.ms': 55_000,
+      # Must be less or equal to `message.timeout.ms` defaults
+      'transaction.timeout.ms': 45_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -44,12 +49,12 @@ module WaterDrop
     # option [Integer] max payload size allowed for delivery to Kafka
     setting :max_payload_size, default: 1_000_012
     # option [Integer] Wait that long for the delivery report or raise an error if this takes
-    #   longer than the timeout.
-    setting :max_wait_timeout, default: 5
+    #   longer than the timeout ms.
+    setting :max_wait_timeout, default: 60_000
     # option [Numeric] how long should we wait between re-checks on the availability of the
     #   delivery report. In a really robust systems, this describes the min-delivery time
     #   for a single sync message when produced in isolation
-    setting :wait_timeout, default: 0.005 # 5 milliseconds
+    setting :wait_timeout, default: 5 # 5 milliseconds
     # option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
     #   we raise an exception.
     #   When this is set to `true`, upon full queue, we won't raise an error. There will be error
@@ -60,12 +65,14 @@ module WaterDrop
     # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
     #   The retry will happen with the same message and backoff should give us some time to
     #   dispatch previously buffered messages.
-    setting :wait_backoff_on_queue_full, default: 0.1
-    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    setting :wait_backoff_on_queue_full, default: 100
+    # option [Numeric] how many ms should we wait with the backoff on queue having space for
     # more messages before re-raising the error.
-    setting :wait_timeout_on_queue_full, default: 10
+    setting :wait_timeout_on_queue_full, default: 10_000
+    # option [Boolean] should we instrument non-critical, retryable queue full errors
+    setting :instrument_on_wait_queue_full, default: true
     # option [Numeric] How long to wait before retrying a retryable transaction related error
-    setting :wait_backoff_on_transaction_command, default: 0.5
+    setting :wait_backoff_on_transaction_command, default: 500
     # option [Numeric] How many times to retry a retryable transaction related error before
     #   giving up
     setting :max_attempts_on_transaction_command, default: 5

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -118,6 +118,13 @@ module WaterDrop
       end
 
       # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closing(event)
+        info(event, 'Closing producer')
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @note While this says "Closing producer", it produces a nice message with time taken:
+      #   "Closing producer took 12 ms" indicating it happened in the past.
       def on_producer_closed(event)
         info(event, 'Closing producer')
       end
@@ -180,7 +187,11 @@ module WaterDrop
       # @param event [Dry::Events::Event] event that happened with the details
       # @param log_message [String] message we want to publish
       def info(event, log_message)
-        @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
+        if event.payload.key?(:time)
+          @logger.info("[#{event[:producer_id]}] #{log_message} took #{event[:time]} ms")
+        else
+          @logger.info("[#{event[:producer_id]}] #{log_message}")
+        end
       end
 
       # @param event [Dry::Events::Event] event that happened with the details

data/lib/waterdrop/instrumentation/notifications.rb

@@ -7,6 +7,8 @@ module WaterDrop
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
       EVENTS = %w[
+        producer.connected
+        producer.closing
         producer.closed
 
         message.produced_async

data/lib/waterdrop/producer/sync.rb

@@ -52,8 +52,8 @@ module WaterDrop
       # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
       #
       # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
-      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
-      #   some handles are still pending
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and some
+      #   handles are still pending
       # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)

data/lib/waterdrop/producer/transactions.rb

@@ -132,8 +132,7 @@ module WaterDrop
             client.send_offsets_to_transaction(
               consumer,
               tpl,
-              # This setting is at the moment in seconds and we require ms
-              @config.max_wait_timeout * 1_000
+              @config.max_wait_timeout
             )
           end
         end
@@ -197,7 +196,7 @@ module WaterDrop
 
         if do_retry
           # Backoff more and more before retries
-          sleep(config.wait_backoff_on_transaction_command * attempt)
+          sleep((config.wait_backoff_on_transaction_command / 1_000.0) * attempt)
 
           retry
         end

data/lib/waterdrop/producer.rb

@@ -117,6 +117,7 @@ module WaterDrop
         )
 
         @status.connected!
+        @monitor.instrument('producer.connected', producer_id: id)
       end
 
       @client
@@ -125,9 +126,10 @@ module WaterDrop
     # Fetches and caches the partition count of a topic
     #
     # @param topic [String] topic for which we want to get the number of partitions
-    # @return [Integer] number of partitions of the requested topic
+    # @return [Integer] number of partitions of the requested topic or -1 if number could not be
+    #   retrieved.
     #
-    # @note It uses the underlying `rdkafka-ruby` partition count cache.
+    # @note It uses the underlying `rdkafka-ruby` partition count fetch and cache.
     def partition_count(topic)
       client.partition_count(topic.to_s)
     end
@@ -159,6 +161,7 @@ module WaterDrop
           producer_id: id
         ) do
           @status.closing!
+          @monitor.instrument('producer.closing', producer_id: id)
 
           # No need for auto-gc if everything got closed by us
           # This should be used only in case a producer was not closed properly and forgotten
@@ -185,8 +188,7 @@ module WaterDrop
             # The linger.ms time will be ignored for the duration of the call,
             # queued messages will be sent to the broker as soon as possible.
             begin
-              # `max_wait_timeout` is in seconds at the moment
-              @client.flush(@config.max_wait_timeout * 1_000) unless @client.closed?
+              @client.flush(@config.max_wait_timeout) unless @client.closed?
             # We can safely ignore timeouts here because any left outstanding requests
             # will anyhow force wait on close if not forced.
             # If forced, we will purge the queue and just close
@@ -247,8 +249,9 @@ module WaterDrop
     # @param handler [Rdkafka::Producer::DeliveryHandle]
     def wait(handler)
       handler.wait(
-        max_wait_timeout: @config.max_wait_timeout,
-        wait_timeout: @config.wait_timeout
+        # rdkafka max_wait_timeout is in seconds and we use ms
+        max_wait_timeout: @config.max_wait_timeout / 1_000.0,
+        wait_timeout: @config.wait_timeout / 1_000.0
       )
     end
 
@@ -283,7 +286,7 @@ module WaterDrop
       # If we're running for longer than the timeout, we need to re-raise the queue full.
       # This will prevent from situation where cluster is down forever and we just retry and retry
       # in an infinite loop, effectively hanging the processing
-      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full * 1_000
+      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
 
       label = caller_locations(2, 1)[0].label.split(' ').last
 
@@ -294,22 +297,28 @@ module WaterDrop
       begin
         raise Errors::ProduceError, e.inspect
       rescue Errors::ProduceError => e
-        # We want to instrument on this event even when we restart it.
-        # The reason is simple: instrumentation and visibility.
-        # We can recover from this, but despite that we should be able to instrument this.
-        # If this type of event happens too often, it may indicate that the buffer settings are not
-        # well configured.
-        @monitor.instrument(
-          'error.occurred',
-          producer_id: id,
-          message: message,
-          error: e,
-          type: "message.#{label}"
-        )
+        # Users can configure this because in pipe-like flows with high throughput, queue full with
+        # retry may be used as a throttling system that will backoff and wait.
+        # In such scenarios this error notification can be removed and until queue full is
+        # retryable, it will not be raised as an error.
+        if @config.instrument_on_wait_queue_full
+          # We want to instrument on this event even when we restart it.
+          # The reason is simple: instrumentation and visibility.
+          # We can recover from this, but despite that we should be able to instrument this.
+          # If this type of event happens too often, it may indicate that the buffer settings are
+          # not well configured.
+          @monitor.instrument(
+            'error.occurred',
+            producer_id: id,
+            message: message,
+            error: e,
+            type: "message.#{label}"
+          )
+        end
 
         # We do not poll the producer because polling happens in a background thread
         # It also should not be a frequent case (queue full), hence it's ok to just throttle.
-        sleep @config.wait_backoff_on_queue_full
+        sleep @config.wait_backoff_on_queue_full / 1_000.0
       end
 
       @operations_in_progress.decrement

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.13'
+  VERSION = '2.7.0.alpha1'
 end

data/waterdrop.gemspec

@@ -16,9 +16,11 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.2.3', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.0.alpha1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
+  spec.required_ruby_version = '>= 3.0.0'
+
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.6.13
+  version: 2.7.0.alpha1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-29 00:00:00.000000000 Z
+date: 2024-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: 2.4.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: 2.4.0.alpha1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -144,7 +144,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="