waterdrop 2.5.2 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 395dba13fcabb99d9aaf8df05f72f2095330e596171106889817dd348b31d615
-  data.tar.gz: 23d6bad1ecf0174fa4d9f5559b7f8b0476398723d57aee4fe9a43e2fa8304501
+  metadata.gz: ed73a2332f0161e71e385fd2250c96bc43383eead9f2945b2580226f855f2643
+  data.tar.gz: 20705696d8534e5b3e10e84a4ae85f05a01025007879b0f59cff7d26f61d28e3
 SHA512:
-  metadata.gz: 49926e4c77977f027f3699bb40292c51643f39a23e62bbcb5778eb5bac9201b63cd2f304ccd6775d271b43205aa38956239951af5aece661526350cfb6cc1f5a
-  data.tar.gz: 9994568f67459cdc2dcd5aec73665bf2d0344782aa3117ffcb0449926909819eacea3afd1f49187937745ed472e34f83854fd3e6a3139081c764f1cad491cd6c
+  metadata.gz: 92533a6e46992a10b2c7d4f3c6cece7f905697cb3c7c2576ec3f0364400b14c71695f1202d8c4b896168753c1d38b888e996632e6894a7cdbdc9f92e940540a6
+  data.tar.gz: '0945428add01cbf32e84c15458db0e39cbc1caabf8e7af9db4221b8e6149162a26946206632858154c202e702c2564a0857d9d420cce3711916f64ccfa1eab87'

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # WaterDrop changelog
 
+### 2.6.0 (2023-06-11)
+- [Improvement] Introduce `client_class` setting for ability to replace underlying client with anything specific to a given env (dev, test, etc).
+- [Improvement] Introduce `Clients::Buffered` useful for writing specs that do not have to talk with Kafka (id-ilych)
+- [Improvement] Make `#produce` method private to avoid confusion and make sure it is not used directly (it is not part of the official API).
+- [Change] Change `wait_on_queue_full` from `false` to `true` as a default.
+- [Change] Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full` to match what it actually does.
+- [Enhancement] Introduce `wait_timeout_on_queue_full` with proper meaning. That is, this represents time after which despite backoff the error will be raised. This should allow to raise an error in case the backoff attempts were insufficient. This prevents from a case, where upon never deliverable messages we would end up with an invite loop.
+- [Fix] Provide `type` for queue full errors that references the appropriate public API method correctly.
+
+### Upgrade notes
+
+1. Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full`.
+2. Set `wait_on_queue_full` to `false` if you did not use it and do not want.
+
+## 2.5.3 (2023-05-26)
+- [Enhancement] Include topic name in the `error.occurred` notification payload.
+- [Enhancement] Include topic name in the `message.acknowledged` notification payload.
+- [Maintenance] Require `karafka-core` `2.0.13`
+
 ## 2.5.2 (2023-04-24)
 - [Fix] Require missing Pathname (#345)
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.5.2)
-      karafka-core (>= 2.0.12, < 3.0.0)
+    waterdrop (2.6.0)
+      karafka-core (>= 2.0.13, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,10 +22,10 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.12)
+    karafka-core (2.0.13)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12.1)
-    karafka-rdkafka (0.12.1)
+      karafka-rdkafka (>= 0.12.3)
+    karafka-rdkafka (0.12.3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/README.md

@@ -93,15 +93,16 @@ end
 
 Some of the options are:
 
-| Option                       | Description                                                      |
-|------------------------------|------------------------------------------------------------------|
-| `id`                         | id of the producer for instrumentation and logging               |
-| `logger`                     | Logger that we want to use                                       |
-| `deliver`                    | Should we send messages to Kafka or just fake the delivery       |
-| `max_wait_timeout`           | Waits that long for the delivery report or raises an error       |
-| `wait_timeout`               | Waits that long before re-check of delivery report availability  |
-| `wait_on_queue_full`         | Should be wait on queue full or raise an error when that happens |
-| `wait_on_queue_full_timeout` | Waits that long before retry when queue is full                  |
+| Option                       | Description                                                                |
+|------------------------------|----------------------------------------------------------------------------|
+| `id`                         | id of the producer for instrumentation and logging                         |
+| `logger`                     | Logger that we want to use                                                 |
+| `deliver`                    | Should we send messages to Kafka or just fake the delivery                 |
+| `max_wait_timeout`           | Waits that long for the delivery report or raises an error                 |
+| `wait_timeout`               | Waits that long before re-check of delivery report availability            |
+| `wait_on_queue_full`         | Should be wait on queue full or raise an error when that happens           |
+| `wait_backoff_on_queue_full` | Waits that long before retry when queue is full                            |
+| `wait_timeout_on_queue_full` | If back-offs and attempts that that much time, error won't be retried more |
 
 Full list of the root configuration options is available [here](https://github.com/karafka/waterdrop/blob/master/lib/waterdrop/config.rb#L25).
 

data/config/locales/errors.yml

@@ -10,6 +10,9 @@ en:
       max_wait_timeout_format: must be an integer that is equal or bigger than 0
       kafka_format: must be a hash with symbol based keys
       kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
+      wait_on_queue_full_format: must be boolean
+      wait_backoff_on_queue_full_format: must be a numeric that is bigger or equal to 0
+      wait_timeout_on_queue_full_format: must be a numeric that is bigger or equal to 0
 
     message:
       missing: must be present

data/lib/waterdrop/clients/buffered.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Clients
+    # Client used to buffer messages that we send out in specs and other places.
+    class Buffered < Clients::Dummy
+      attr_accessor :messages
+
+      # Sync fake response for the message delivery to Kafka, since we do not dispatch anything
+      class SyncResponse
+        # @param _args Handler wait arguments (irrelevant as waiting is fake here)
+        def wait(*_args)
+          false
+        end
+      end
+
+      # @param args [Object] anything accepted by `Clients::Dummy`
+      def initialize(*args)
+        super
+        @messages = []
+        @topics = Hash.new { |k, v| k[v] = [] }
+      end
+
+      # "Produces" message to Kafka: it acknowledges it locally, adds it to the internal buffer
+      # @param message [Hash] `WaterDrop::Producer#produce_sync` message hash
+      def produce(message)
+        topic = message.fetch(:topic) { raise ArgumentError, ':topic is missing' }
+        @topics[topic] << message
+        @messages << message
+        SyncResponse.new
+      end
+
+      # Returns messages produced to a given topic
+      # @param topic [String]
+      def messages_for(topic)
+        @topics[topic]
+      end
+
+      # Clears internal buffer
+      # Used in between specs so messages do not leak out
+      def reset
+        @messages.clear
+        @topics.each_value(&:clear)
+      end
+    end
+  end
+end

data/lib/waterdrop/{producer/dummy_client.rb → clients/dummy.rb}

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 module WaterDrop
-  class Producer
+  module Clients
     # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
-    # want to dispatch anything to Kafka
-    class DummyClient
-      # @return [DummyClient] dummy instance
-      def initialize
+    # want to dispatch anything to Kafka.
+    #
+    # It does not store anything and just ignores messages.
+    class Dummy
+      # @param _producer [WaterDrop::Producer]
+      # @return [Dummy] dummy instance
+      def initialize(_producer)
         @counter = -1
       end
 

data/lib/waterdrop/clients/rdkafka.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the clients that WaterDrop may use under the hood
+  module Clients
+    # Default Rdkafka client.
+    # Since we use the ::Rdkafka::Producer under the hood, this is just a module that aligns with
+    # client building API for the convenience.
+    module Rdkafka
+      class << self
+        # @param producer [WaterDrop::Producer] producer instance with its config, etc
+        # @note We overwrite this that way, because we do not care
+        def new(producer)
+          client = ::Rdkafka::Config.new(producer.config.kafka.to_h).producer
+
+          # This callback is not global and is per client, thus we do not have to wrap it with a
+          # callbacks manager to make it work
+          client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
+            producer.id,
+            producer.config.monitor
+          )
+
+          client
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/config.rb

@@ -56,15 +56,20 @@ module WaterDrop
     #   in the `error.occurred` notification pipeline with a proper type as while this is
     #   recoverable, in a high number it still may mean issues.
     #   Waiting is one of the recommended strategies.
-    setting :wait_on_queue_full, default: false
+    setting :wait_on_queue_full, default: true
     # option [Integer] how long (in seconds) should we backoff before a retry when queue is full
     #   The retry will happen with the same message and backoff should give us some time to
     #   dispatch previously buffered messages.
-    setting :wait_on_queue_full_timeout, default: 0.1
+    setting :wait_backoff_on_queue_full, default: 0.1
+    # option [Numeric] how many seconds should we wait with the backoff on queue having space for
+    # more messages before re-raising the error.
+    setting :wait_timeout_on_queue_full, default: 10
     # option [Boolean] should we send messages. Setting this to false can be really useful when
     #   testing and or developing because when set to false, won't actually ping Kafka but will
     #   run all the validations, etc
     setting :deliver, default: true
+    # option [Class] class for usage when creating the underlying client used to dispatch messages
+    setting :client_class, default: Clients::Rdkafka
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}

data/lib/waterdrop/contracts/config.rb

@@ -19,6 +19,9 @@ module WaterDrop
       required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:wait_timeout) { |val| val.is_a?(Numeric) && val.positive? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
 
       # rdkafka allows both symbols and strings as keys for config but then casts them to strings
       # This can be confusing, so we expect all keys to be symbolized

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -35,6 +35,7 @@ module WaterDrop
             producer_id: @producer_id,
             offset: delivery_report.offset,
             partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
             type: 'librdkafka.dispatch_error'
           )
         end
@@ -45,7 +46,8 @@ module WaterDrop
             'message.acknowledged',
             producer_id: @producer_id,
             offset: delivery_report.offset,
-            partition: delivery_report.partition
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name
           )
         end
       end

data/lib/waterdrop/producer/builder.rb

@@ -10,18 +10,13 @@ module WaterDrop
       # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
       #   when we don't want to dispatch any messages
       def call(producer, config)
-        return DummyClient.new unless config.deliver
+        klass = config.client_class
+        # This allows us to have backwards compatibility.
+        # If it is the default client and delivery is set to false, we use dummy as we used to
+        # before `client_class` was introduced
+        klass = Clients::Dummy if klass == Clients::Rdkafka && !config.deliver
 
-        client = Rdkafka::Config.new(config.kafka.to_h).producer
-
-        # This callback is not global and is per client, thus we do not have to wrap it with a
-        # callbacks manager to make it work
-        client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
-          producer.id,
-          config.monitor
-        )
-
-        client
+        klass.new(producer)
       end
     end
   end

data/lib/waterdrop/producer.rb

@@ -7,6 +7,7 @@ module WaterDrop
     include Sync
     include Async
     include Buffer
+    include ::Karafka::Core::Helpers::Time
 
     # Which of the inline flow errors do we want to intercept and re-bind
     SUPPORTED_FLOW_ERRORS = [
@@ -168,15 +169,35 @@ module WaterDrop
       @contract.validate!(message, Errors::MessageInvalidError)
     end
 
+    # Waits on a given handler
+    #
+    # @param handler [Rdkafka::Producer::DeliveryHandle]
+    def wait(handler)
+      handler.wait(
+        max_wait_timeout: @config.max_wait_timeout,
+        wait_timeout: @config.wait_timeout
+      )
+    end
+
+    private
+
     # Runs the client produce method with a given message
     #
     # @param message [Hash] message we want to send
     def produce(message)
+      produce_time ||= monotonic_now
+
       client.produce(**message)
     rescue SUPPORTED_FLOW_ERRORS.first => e
       # Unless we want to wait and retry and it's a full queue, we raise normally
       raise unless @config.wait_on_queue_full
       raise unless e.code == :queue_full
+      # If we're running for longer than the timeout, we need to re-raise the queue full.
+      # This will prevent from situation where cluster is down forever and we just retry and retry
+      # in an infinite loop, effectively hanging the processing
+      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full * 1_000
+
+      label = caller_locations(2, 1)[0].label.split(' ').last
 
       # We use this syntax here because we want to preserve the original `#cause` when we
       # instrument the error and there is no way to manually assign `#cause` value. We want to keep
@@ -195,25 +216,15 @@ module WaterDrop
           producer_id: id,
           message: message,
           error: e,
-          type: 'message.produce'
+          type: "message.#{label}"
         )
 
         # We do not poll the producer because polling happens in a background thread
         # It also should not be a frequent case (queue full), hence it's ok to just throttle.
-        sleep @config.wait_on_queue_full_timeout
+        sleep @config.wait_backoff_on_queue_full
       end
 
       retry
     end
-
-    # Waits on a given handler
-    #
-    # @param handler [Rdkafka::Producer::DeliveryHandle]
-    def wait(handler)
-      handler.wait(
-        max_wait_timeout: @config.max_wait_timeout,
-        wait_timeout: @config.wait_timeout
-      )
-    end
   end
 end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.5.2'
+  VERSION = '2.6.0'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.5.2
+  version: 2.6.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-04-24 00:00:00.000000000 Z
+date: 2023-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
+        version: 2.0.13
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -95,6 +95,9 @@ files:
 - config/locales/errors.yml
 - docker-compose.yml
 - lib/waterdrop.rb
+- lib/waterdrop/clients/buffered.rb
+- lib/waterdrop/clients/dummy.rb
+- lib/waterdrop/clients/rdkafka.rb
 - lib/waterdrop/config.rb
 - lib/waterdrop/contracts.rb
 - lib/waterdrop/contracts/config.rb
@@ -116,7 +119,6 @@ files:
 - lib/waterdrop/producer/async.rb
 - lib/waterdrop/producer/buffer.rb
 - lib/waterdrop/producer/builder.rb
-- lib/waterdrop/producer/dummy_client.rb
 - lib/waterdrop/producer/status.rb
 - lib/waterdrop/producer/sync.rb
 - lib/waterdrop/version.rb