waterdrop 2.0.7 → 2.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 261d7b8295e2e1b3a6b9658d4409dc58ff8c336e8e5d6064e80d92c8e3215bbe
-  data.tar.gz: f67bad60fe2cd37ac1658533ceec0daa1ec9ab527bb022da7f1cc40ccde89088
+  metadata.gz: 3798d61ff3464dd6aaeb21f5457230a3b079748f260c59138142144efc62dc40
+  data.tar.gz: c9699bc202a67fbb8e456a4e0d316f56e8635e0986a09d0429a25999553c4a76
 SHA512:
-  metadata.gz: e0529057c22d4b76c61f788f038e8fb024785aa3a2c89c266cfe2b50d1f2b44e8a96e16ca626f62e721d4ae48fe4afac7113e32e5cd5ccbc5e7adf36aa0558af
-  data.tar.gz: 31241cad722c315aefab9d4ad2613908f053bf3c905779e8fe5c6f1975fbc866b34526442386c1faec2e2386d5f5332c4b7e80a326aa8aca9b8ab1b94e51d572
+  metadata.gz: c6a6d03d205314775b71ca0c4e922ef52159965e30b9280727712f25368f6af78c821d6d468b2362bd4094fbd2d4747195dc1e3a7cd76a0e4582f81a66829b43
+  data.tar.gz: 9541ae05afde65cf1124661fd57e23afb4265601748c6b4a60f6fc6f8cabe463cf59b6ac683e7d047e454cc671182808f800add0f68b780bbb1b1325abcab892

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/.github/workflows/ci.yml

@@ -1,5 +1,7 @@
 name: ci
 
+concurrency: ci-${{ github.ref }}
+
 on:
   pull_request:
   push:
@@ -14,35 +16,44 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
+          - '3.2'
+          - '3.1'
           - '3.0'
           - '2.7'
-          - '2.6'
-          - 'jruby-9.3.1.0'
         include:
-          - ruby: '3.0'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
+
       - name: Run Kafka with docker-compose
-        # We need to give Kafka enough time to start and create all the needed topics, etc
-        # If anyone has a better idea on how to do it smart and easily, please contact me
         run: |
-          docker-compose up -d
-          sleep 5
+          docker-compose up -d || (sleep 5 && docker-compose up -d)
+
+      - name: Wait for Kafka
+        run: |
+          sleep 15
+
       - name: Install latest bundler
         run: |
           gem install bundler --no-document
           bundle config set without 'tools benchmarks docs'
+
       - name: Bundle install
         run: |
           bundle config set without development
           bundle install --jobs 4 --retry 3
+
       - name: Run all tests
         env:
           GITHUB_COVERAGE: ${{matrix.coverage}}
@@ -53,13 +64,13 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.0
+          ruby-version: 3.2
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin
@@ -72,7 +83,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Run Coditsu

data/.ruby-version

@@ -1 +1 @@
-3.0.2
+3.2.2

data/CHANGELOG.md

@@ -1,5 +1,205 @@
 # WaterDrop changelog
 
+## 2.6.11 (2023-10-25)
+- [Enhancement] Return delivery handles and delivery report for both dummy and buffered clients with proper topics, partitions and offsets assign and auto-increment offsets per partition.
+- [Fix] Fix a case where buffered test client would not accumulate messages on failed transactions
+
+## 2.6.10 (2023-10-24)
+- [Improvement] Introduce `message.purged` event to indicate that a message that was not delivered to Kafka was purged. This most of the time refers to messages that were part of a transaction and were not yet dispatched to Kafka. It always means, that given message was not delivered but in case of transactions it is expected. In case of non-transactional it usually means `#purge` usage or exceeding `message.timeout.ms` so `librdkafka` removes this message from its internal queue. Non-transactional producers do **not** use this and pipe purges to `error.occurred`.
+- [Fix] Fix a case where `message.acknowledged` would not have `caller` key.
+- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
+
+## 2.6.9 (2023-10-23)
+- [Improvement] Introduce a `transaction.finished` event to indicate that transaction has finished whether it was aborted or committed.
+- [Improvement] Use `transaction.committed` event to indicate that transaction has been committed.
+
+## 2.6.8 (2023-10-20)
+- **[Feature]** Introduce transactions support.
+- [Improvement] Expand `LoggerListener` to inform about transactions (info level).
+- [Improvement] Allow waterdrop to use topic as a symbol or a string.
+- [Improvement] Enhance both `message.acknowledged` and `error.occurred` (for `librdkafka.dispatch_error`) with full delivery_report.
+- [Improvement] Provide `#close!` that will force producer close even with outgoing data after the ma wait timeout.
+- [Improvement] Provide `#purge` that will purge any outgoing data and data from the internal queues (both WaterDrop and librdkafka).
+- [Fix] Fix the `librdkafka.dispatch_error` error dispatch for errors with negative code.
+
+## 2.6.7 (2023-09-01)
+- [Improvement] early flush data from `librdkafka` internal buffer before closing.
+- [Maintenance] Update the signing cert as the old one expired.
+
+## 2.6.6 (2023-08-03)
+- [Improvement] Provide `log_messages` option to `LoggerListener` so the extensive messages data logging can disabled.
+
+## 2.6.5 (2023-07-22)
+- [Fix] Add cause to the errors that are passed into instrumentation (konalegi)
+
+## 2.6.4 (2023-07-11)
+- [Improvement] Use original error `#inspect` for `WaterDrop::Errors::ProduceError` and `WaterDrop::Errors::ProduceManyError` instead of the current empty string.
+
+## 2.6.3 (2023-06-28)
+- [Change] Use `Concurrent::AtomicFixnum` to track operations in progress to prevent potential race conditions on JRuby and TruffleRuby (not yet supported but this is for future usage).
+- [Change] Require `karafka-rdkafka` `>= 0.13.2`.
+- [Change] Require 'karafka-core' `>= 2.1.1`
+
+## 2.6.2 (2023-06-21)
+- [Refactor] Introduce a counter-based locking approach to make sure, that we close the producer safely but at the same time not to limit messages production with producing lock.
+- [Refactor] Make private methods private.
+- [Refactor] Validate that producer is not closed only when attempting to produce.
+- [Refactor] Improve one 5 minute long spec to run in 10 seconds.
+- [Refactor] clear client assignment after closing.
+
+## 2.6.1 (2023-06-19)
+- [Refactor] Remove no longer needed patches.
+- [Fix] Fork detection on a short lived processes seems to fail. Clear the used parent process client reference not to close it in the finalizer (#356).
+- [Change] Require `karafka-rdkafka` `>= 0.13.0`.
+- [Change] Require 'karafka-core' `>= 2.1.0`
+
+## 2.6.0 (2023-06-11)
+- [Improvement] Introduce `client_class` setting for ability to replace underlying client with anything specific to a given env (dev, test, etc).
+- [Improvement] Introduce `Clients::Buffered` useful for writing specs that do not have to talk with Kafka (id-ilych)
+- [Improvement] Make `#produce` method private to avoid confusion and make sure it is not used directly (it is not part of the official API).
+- [Change] Change `wait_on_queue_full` from `false` to `true` as a default.
+- [Change] Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full` to match what it actually does.
+- [Enhancement] Introduce `wait_timeout_on_queue_full` with proper meaning. That is, this represents time after which despite backoff the error will be raised. This should allow to raise an error in case the backoff attempts were insufficient. This prevents from a case, where upon never deliverable messages we would end up with an infinite loop.
+- [Fix] Provide `type` for queue full errors that references the appropriate public API method correctly.
+
+### Upgrade notes
+
+1. Rename `wait_on_queue_full_timeout` to `wait_backoff_on_queue_full`.
+2. Set `wait_on_queue_full` to `false` if you did not use it and do not want.
+
+## 2.5.3 (2023-05-26)
+- [Enhancement] Include topic name in the `error.occurred` notification payload.
+- [Enhancement] Include topic name in the `message.acknowledged` notification payload.
+- [Maintenance] Require `karafka-core` `2.0.13`
+
+## 2.5.2 (2023-04-24)
+- [Fix] Require missing Pathname (#345)
+
+## 2.5.1 (2023-03-09)
+- [Feature] Introduce a configurable backoff upon `librdkafka` queue full (false by default).
+
+## 2.5.0 (2023-03-04)
+- [Feature] Pipe **all** the errors including synchronous errors via the `error.occurred`.
+- [Improvement] Pipe delivery errors that occurred not via the error callback using the `error.occurred` channel.
+- [Improvement] Introduce `WaterDrop::Errors::ProduceError` and `WaterDrop::Errors::ProduceManyError` for any inline raised errors that occur. You can get the original error by using the `#cause`.
+- [Improvement] Include `#dispatched` messages handler in the `WaterDrop::Errors::ProduceManyError` error, to be able to understand which of the messages were delegated to `librdkafka` prior to the failure.
+- [Maintenance] Remove the `WaterDrop::Errors::FlushFailureError` in favour of correct error that occurred to unify the error handling.
+- [Maintenance] Rename `Datadog::Listener` to `Datadog::MetricsListener` to align with Karafka (#329).
+- [Fix] Do **not** flush when there is no data to flush in the internal buffer.
+- [Fix] Wait on the final data flush for short-lived producers to make sure, that the message is actually dispatched by `librdkafka` or timeout.
+
+### Upgrade notes
+
+Please note, this **is** a **breaking** release, hence `2.5.0`.
+
+1. If you used to catch `WaterDrop::Errors::FlushFailureError` now you need to catch `WaterDrop::Errors::ProduceError`. `WaterDrop::Errors::ProduceManyError` is based on the `ProduceError`, hence it should be enough.
+2. Prior to `2.5.0` there was always a chance of partial dispatches via `produce_many_` methods. Now you can get the info on all the errors via `error.occurred`.
+3. Inline `Rdkafka::RdkafkaError` are now re-raised via `WaterDrop::Errors::ProduceError` and available under `#cause`. Async `Rdkafka::RdkafkaError` errors are still directly available and you can differentiate between errors using the event `type`.
+4. If you are using the Datadog listener, you need to:
+
+```ruby
+# Replace require:
+require 'waterdrop/instrumentation/vendors/datadog/listener'
+# With
+require 'waterdrop/instrumentation/vendors/datadog/metrics_listener'
+
+# Replace references of
+::WaterDrop::Instrumentation::Vendors::Datadog::Listener.new
+# With
+::WaterDrop::Instrumentation::Vendors::Datadog::MetricsListener.new
+```
+
+## 2.4.11 (2023-02-24)
+- Replace the local rspec locator with generalized core one.
+- Make `::WaterDrop::Instrumentation::Notifications::EVENTS` list public for anyone wanting to re-bind those into a different notification bus.
+
+## 2.4.10 (2023-01-30)
+- Include `caller` in the error instrumentation to align with Karafka.
+
+## 2.4.9 (2023-01-11)
+- Remove empty debug logging out of `LoggerListener`.
+- Do not lock Ruby version in Karafka in favour of `karafka-core`.
+- Make sure `karafka-core` version is at least `2.0.9` to make sure we run `karafka-rdkafka`.
+
+## 2.4.8 (2023-01-07)
+- Use monotonic time from Karafka core.
+
+## 2.4.7 (2022-12-18)
+- Add support to customizable middlewares that can modify message hash prior to validation and dispatch.
+- Fix a case where upon not-available leader, metadata request would not be retried
+- Require `karafka-core` 2.0.7.
+
+## 2.4.6 (2022-12-10)
+- Set `statistics.interval.ms` to 5 seconds by default, so the defaults cover all the instrumentation out of the box.
+
+### Upgrade notes
+
+If you want to disable `librdkafka` statistics because you do not use them at all, update the `kafka` `statistics.interval.ms` setting and set it to `0`:
+
+```ruby
+producer = WaterDrop::Producer.new
+
+producer.setup do |config|
+  config.deliver = true
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    'statistics.interval.ms': 0
+  }
+end
+```
+
+## 2.4.5 (2022-12-10)
+- Fix invalid error scope visibility.
+- Cache partition count to improve messages production and lower stress on Kafka when `partition_key` is on.
+
+## 2.4.4 (2022-12-09)
+- Add temporary patch on top of `rdkafka-ruby` to mitigate metadata fetch timeout failures.
+
+## 2.4.3 (2022-12-07)
+- Support for librdkafka 0.13
+- Update Github Actions
+- Change auto-generated id from `SecureRandom#uuid` to `SecureRandom#hex(6)`
+- Remove shared components that were moved to `karafka-core` from WaterDrop
+
+## 2.4.2 (2022-09-29)
+- Allow sending tombstone messages (#267)
+
+## 2.4.1 (2022-08-01)
+- Replace local statistics decorator with the one extracted to `karafka-core`.
+
+## 2.4.0 (2022-07-28)
+- Small refactor of the DataDog/Statsd listener to align for future extraction to `karafka-common`.
+- Replace `dry-monitor` with home-brew notification layer (API compatible) and allow for usage with `ActiveSupport::Notifications`.
+- Remove all the common code into `karafka-core` and add it as a dependency.
+
+## 2.3.3 (2022-07-18)
+- Replace `dry-validation` with home-brew validation layer and drop direct dependency on `dry-validation`.
+- Remove indirect dependency on dry-configurable from DataDog listener (no changes required).
+
+## 2.3.2 (2022-07-17)
+- Replace `dry-configurable` with home-brew config and drop direct dependency on `dry-configurable`.
+
+## 2.3.1 (2022-06-17)
+- Update rdkafka patches to align with `0.12.0` and `0.11.1` support.
+
+## 2.3.0 (2022-04-03)
+-  Rename StdoutListener to LoggerListener (#240)
+
+## 2.2.0 (2022-02-18)
+- Add Datadog listener for metrics + errors publishing
+- Add Datadog example dashboard template
+- Update Readme to show Dd instrumentation usage
+- Align the directory namespace convention with gem name (waterdrop => WaterDrop)
+- Introduce a common base for validation contracts
+- Drop CI support for ruby 2.6
+- Require all `kafka` settings to have symbol keys (compatibility with Karafka 2.0 and rdkafka)
+
+## 2.1.0 (2022-01-03)
+- Ruby 3.1 support
+- Change the error notification key from `error.emitted` to `error.occurred`.
+- Normalize error tracking and make all the places publish errors into the same notification endpoint (`error.occurred`).
+- Start semantic versioning WaterDrop.
+
 ## 2.0.7 (2021-12-03)
 - Source code metadata url added to the gemspec
 - Replace `:producer` with `:producer_id` in events and update `StdoutListener` accordingly. This change aligns all the events in terms of not publishing the whole producer object in the events.

data/Gemfile

@@ -6,8 +6,6 @@ plugin 'diffend'
 
 gemspec
 
-gem 'rdkafka'
-
 group :development do
   gem 'byebug'
 end

data/Gemfile.lock

@@ -1,112 +1,69 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.0.7)
-      concurrent-ruby (>= 1.1)
-      dry-configurable (~> 0.13)
-      dry-monitor (~> 0.5)
-      dry-validation (~> 1.7)
-      rdkafka (>= 0.10)
+    waterdrop (2.6.11)
+      karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (7.0.7.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
+    concurrent-ruby (1.2.2)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    dry-configurable (0.13.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.6)
-    dry-container (0.9.0)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-    dry-core (0.7.1)
-      concurrent-ruby (~> 1.0)
-    dry-events (0.3.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
-    dry-inflector (0.2.1)
-    dry-initializer (3.0.4)
-    dry-logic (1.2.0)
-      concurrent-ruby (~> 1.0)
-      dry-core (~> 0.5, >= 0.5)
-    dry-monitor (0.5.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-      dry-core (~> 0.5, >= 0.5)
-      dry-events (~> 0.2)
-    dry-schema (1.8.0)
-      concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-logic (~> 1.0)
-      dry-types (~> 1.5)
-    dry-types (1.5.1)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.3)
-      dry-core (~> 0.5, >= 0.5)
-      dry-inflector (~> 0.1, >= 0.1.2)
-      dry-logic (~> 1.0, >= 1.0.2)
-    dry-validation (1.7.0)
-      concurrent-ruby (~> 1.0)
-      dry-container (~> 0.7, >= 0.7.1)
-      dry-core (~> 0.5, >= 0.5)
-      dry-initializer (~> 3.0)
-      dry-schema (~> 1.8, >= 1.8.0)
-    factory_bot (6.2.0)
+    factory_bot (6.3.0)
       activesupport (>= 5.0.0)
-    ffi (1.15.4)
-    i18n (1.8.11)
+    ffi (1.15.5)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    mini_portile2 (2.7.1)
-    minitest (5.14.4)
-    rake (13.0.6)
-    rdkafka (0.11.0)
+    karafka-core (2.2.3)
+      concurrent-ruby (>= 1.1)
+      karafka-rdkafka (>= 0.13.6, < 0.14.0)
+    karafka-rdkafka (0.13.6)
       ffi (~> 1.15)
-      mini_portile2 (~> 2.7)
+      mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    mini_portile2 (2.8.2)
+    minitest (5.19.0)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.3)
-    simplecov (0.21.2)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.6.8)
 
 PLATFORMS
-  x86_64-darwin
   x86_64-linux
 
 DEPENDENCIES
   byebug
   factory_bot
-  rdkafka
   rspec
   simplecov
   waterdrop!
 
 BUNDLED WITH
-   2.2.31
+   2.4.12