waterdrop 2.8.0 → 2.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96435533f730edae9dcf7bed5426641a33715e87b55c804168fa6a02465f1431
-  data.tar.gz: 3737e88ec3ffaa407b1cdaa523cf1d24a25aa85c4d90989a249381dbfb7a80c2
+  metadata.gz: f1dd1ecba0be29bdf637575f0fe8eaea654900f7466d5840c62fbcdeb5000492
+  data.tar.gz: 663b683a670ec1df3700dd3440c80435b376c493bcdb7d024eedcc9a9c7b0de9
 SHA512:
-  metadata.gz: 1b83e8505ca1aa9be5c137c06af1b98e10bb8f39af93e77aefa42302bf8dc2bd95f1442eb2569d5f032e80ac0374b8013e774d1f36494d23e20af0766094e2b3
-  data.tar.gz: 198ea59a4f7ca6a63f9816e7026c97152aaed5e6154142c47e6e96c16696a2a3d4c60cd56bf72b407b092682e9d15c08f1adfa552eb5bdc17a308f77f7424dfc
+  metadata.gz: 8ff3c1d142dd9d855c7a26223ceefa973764dfde23b0ec8530999e87d52b7827b7180b30dd1f9f545a65eb7e1b00a8ddf753baa4fa365b93ec997450491ddc9b
+  data.tar.gz: 0fee34d4935ee757fef55ae9205570894b20894b839a1e6abd80fa918eb7d733603a0c791b3d81c73fd913699f07a1ddf1d3af9921722713accdb8bc90a17255

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug Report
+about: Report an issue within the Karafka ecosystem you've discovered.
+---
+
+To make this process smoother for everyone involved, please read the following information before filling out the template.
+
+Scope of the OSS Support
+===========
+
+We do not provide OSS support for outdated versions of Karafka and its components.
+
+Please ensure that you are using a version that is still actively supported. We cannot assist with any no longer maintained versions unless you support us with our Pro offering (https://karafka.io/docs/Pro-Support/).
+
+We acknowledge that understanding the specifics of your application and its configuration can be essential for resolving certain issues. However, due to the extensive time and resources such analysis can require, this may fall beyond our Open Source Support scope.
+
+If Karafka or its components are critical to your infrastructure, we encourage you to consider our Pro Offering.
+
+By backing us up, you can gain direct assistance and ensure your use case receives the dedicated attention it deserves.
+
+
+Important Links to Read
+===========
+
+Please take a moment to review the following resources before submitting your report:
+
+- Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
+- Support Policy: https://karafka.io/docs/Support/
+- Versions, Lifecycle, and EOL: https://karafka.io/docs/Versions-Lifecycle-and-EOL/
+
+
+Bug Report Details
+===========
+
+Please provide all the details per our Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
+
+Failing to provide the required details may result in the issue being closed. Please include all necessary information to help us understand and resolve your issue effectively.
+
+
+Additional Context
+===========
+
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature Request
+about: Suggest new WaterDrop features or improvements to existing features.
+---
+
+## Is your feature request related to a problem? Please describe.
+
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+
+Add any other context or screenshots about the feature request here.

data/.github/workflows/ci.yml

@@ -18,12 +18,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
         include:
-          - ruby: '3.3'
+          - ruby: '3.4'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4
@@ -63,6 +63,9 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
         run: bundle exec rspec
 
+      - name: Check Kafka logs for unexpected warnings
+        run: bin/verify_kafka_warnings
+
   diffend:
     runs-on: ubuntu-latest
     strategy:
@@ -74,7 +77,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.3
+          ruby-version: 3.4
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin

data/.ruby-version

@@ -1 +1 @@
-3.3.5
+3.4.1

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # WaterDrop changelog
 
+## 2.8.2 (2025-02-13)
+- [Feature] Allow for tagging of producer instances similar to how consumers can be tagged.
+- [Refactor] Ensure all test topics in the test suite start with "it-" prefix. 
+
+## 2.8.1 (2024-12-26)
+- [Enhancement] Raise `WaterDrop::ProducerNotTransactionalError` when attempting to use transactions on a non-transactional producer.
+- [Fix] Disallow closing a producer from within a transaction.
+- [Fix] WaterDrop should prevent opening a transaction using a closed producer.
+
 ## 2.8.0 (2024-09-16)
 
 This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.8.0)
+    waterdrop (2.8.2)
       karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.17.5)
       zeitwerk (~> 2.3)
@@ -9,8 +9,9 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.2.1)
+    activesupport (7.2.2.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
@@ -21,17 +22,18 @@ GEM
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
     base64 (0.2.0)
-    bigdecimal (3.1.8)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.3.4)
+    concurrent-ruby (1.3.5)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
     docile (1.4.1)
     drb (2.2.1)
-    factory_bot (6.5.0)
-      activesupport (>= 5.0.0)
+    factory_bot (6.5.1)
+      activesupport (>= 6.1.0)
     ffi (1.17.0)
-    i18n (1.14.5)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.4)
       karafka-rdkafka (>= 0.15.0, < 0.18.0)
@@ -39,10 +41,10 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    logger (1.6.1)
+    logger (1.6.5)
     mini_portile2 (2.8.7)
-    minitest (5.25.1)
-    ostruct (0.6.0)
+    minitest (5.25.4)
+    ostruct (0.6.1)
     rake (13.2.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -57,7 +59,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.1)
-    securerandom (0.3.1)
+    securerandom (0.3.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/bin/verify_kafka_warnings

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+   "Performing controller activation"
+   "registered with feature metadata.version"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep WARN)
+exit_code=0
+
+while IFS= read -r line; do
+   allowed=0
+   for pattern in "${allowed_patterns[@]}"; do
+       if echo "$line" | grep -q "$pattern"; then
+           allowed=1
+           break
+       fi
+   done
+
+   if [ $allowed -eq 0 ]; then
+       echo "Unexpected warning found:"
+       echo "$line"
+       exit_code=1
+   fi
+done <<< "$warnings"
+
+exit $exit_code

data/docker-compose.yml

@@ -1,9 +1,7 @@
-version: '2'
-
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.1
+    image: confluentinc/cp-kafka:7.8.1
 
     ports:
       - 9092:9092

data/lib/waterdrop/errors.rb

@@ -25,12 +25,18 @@ module WaterDrop
     # Raised when there was an attempt to use a closed producer
     ProducerClosedError = Class.new(BaseError)
 
+    # Raised if you attempt to close the producer from within a transaction. This is not allowed.
+    ProducerTransactionalCloseAttemptError = Class.new(BaseError)
+
     # Raised when we want to send a message that is invalid (impossible topic, etc)
     MessageInvalidError = Class.new(BaseError)
 
     # Raised when we want to commit transactional offset and the input is invalid
     TransactionalOffsetInvalidError = Class.new(BaseError)
 
+    # Raised when transaction attempt happens on a non-transactional producer
+    ProducerNotTransactionalError = Class.new(BaseError)
+
     # Raised when we've got an unexpected status. This should never happen. If it does, please
     # contact us as it is an error.
     StatusInvalidError = Class.new(BaseError)

data/lib/waterdrop/producer/transactions.rb

@@ -55,11 +55,20 @@ module WaterDrop
       #
       #   handler.wait
       def transaction
+        unless transactional?
+          raise(
+            Errors::ProducerNotTransactionalError,
+            "#{id} is not transactional"
+          )
+        end
+
         # This will safely allow us to support one operation transactions so a transactional
         # producer can work without the transactional block if needed
         return yield if @transaction_mutex.owned?
 
         @transaction_mutex.synchronize do
+          ensure_active!
+
           transactional_instrument(:finished) do
             with_transactional_error_handling(:begin) do
               transactional_instrument(:started) { client.begin_transaction }

data/lib/waterdrop/producer.rb

@@ -9,6 +9,7 @@ module WaterDrop
     include Buffer
     include Transactions
     include ::Karafka::Core::Helpers::Time
+    include ::Karafka::Core::Taggable
 
     # Local storage for given thread waterdrop client references for variants
     ::Fiber.send(:attr_accessor, :waterdrop_clients)
@@ -85,6 +86,7 @@ module WaterDrop
 
       # Don't allow to obtain a client reference for a producer that was not configured
       raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closed?
 
       @connecting_mutex.synchronize do
         return @client if @client && @pid == Process.pid
@@ -180,65 +182,75 @@ module WaterDrop
     # @param force [Boolean] should we force closing even with outstanding messages after the
     #   max wait timeout
     def close(force: false)
-      @operating_mutex.synchronize do
-        return unless @status.active?
-
-        @monitor.instrument(
-          'producer.closed',
-          producer_id: id
-        ) do
-          @status.closing!
-          @monitor.instrument('producer.closing', producer_id: id)
-
-          # No need for auto-gc if everything got closed by us
-          # This should be used only in case a producer was not closed properly and forgotten
-          ObjectSpace.undefine_finalizer(id)
-
-          # We save this thread id because we need to bypass the activity verification on the
-          # producer for final flush of buffers.
-          @closing_thread_id = Thread.current.object_id
-
-          # Wait until all the outgoing operations are done. Only when no one is using the
-          # underlying client running operations we can close
-          sleep(0.001) until @operations_in_progress.value.zero?
-
-          # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
-          # as we close the client after the flushing (even if blocked by the mutex)
-          flush(true)
-
-          # We should not close the client in several threads the same time
-          # It is safe to run it several times but not exactly the same moment
-          # We also mark it as closed only if it was connected, if not, it would trigger a new
-          # connection that anyhow would be immediately closed
-          if @client
-            # Why do we trigger it early instead of just having `#close` do it?
-            # The linger.ms time will be ignored for the duration of the call,
-            # queued messages will be sent to the broker as soon as possible.
-            begin
-              @client.flush(current_variant.max_wait_timeout) unless @client.closed?
-            # We can safely ignore timeouts here because any left outstanding requests
-            # will anyhow force wait on close if not forced.
-            # If forced, we will purge the queue and just close
-            rescue ::Rdkafka::RdkafkaError, Rdkafka::AbstractHandle::WaitTimeoutError
-              nil
-            ensure
-              # Purge fully the local queue in case of a forceful shutdown just to be sure, that
-              # there are no dangling messages. In case flush was successful, there should be
-              # none but we do it just in case it timed out
-              purge if force
-            end
+      # If we already own the transactional mutex, it means we are inside of a transaction and
+      # it should not we allowed to close the producer in such a case.
+      if @transaction_mutex.locked? && @transaction_mutex.owned?
+        raise Errors::ProducerTransactionalCloseAttemptError, id
+      end
 
-            @client.close
+      # The transactional mutex here can be used even when no transactions are in use
+      # It prevents us from closing a mutex during transactions and is irrelevant in other cases
+      @transaction_mutex.synchronize do
+        @operating_mutex.synchronize do
+          return unless @status.active?
 
-            @client = nil
-          end
+          @monitor.instrument(
+            'producer.closed',
+            producer_id: id
+          ) do
+            @status.closing!
+            @monitor.instrument('producer.closing', producer_id: id)
+
+            # No need for auto-gc if everything got closed by us
+            # This should be used only in case a producer was not closed properly and forgotten
+            ObjectSpace.undefine_finalizer(id)
+
+            # We save this thread id because we need to bypass the activity verification on the
+            # producer for final flush of buffers.
+            @closing_thread_id = Thread.current.object_id
+
+            # Wait until all the outgoing operations are done. Only when no one is using the
+            # underlying client running operations we can close
+            sleep(0.001) until @operations_in_progress.value.zero?
+
+            # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
+            # as we close the client after the flushing (even if blocked by the mutex)
+            flush(true)
+
+            # We should not close the client in several threads the same time
+            # It is safe to run it several times but not exactly the same moment
+            # We also mark it as closed only if it was connected, if not, it would trigger a new
+            # connection that anyhow would be immediately closed
+            if @client
+              # Why do we trigger it early instead of just having `#close` do it?
+              # The linger.ms time will be ignored for the duration of the call,
+              # queued messages will be sent to the broker as soon as possible.
+              begin
+                @client.flush(current_variant.max_wait_timeout) unless @client.closed?
+              # We can safely ignore timeouts here because any left outstanding requests
+              # will anyhow force wait on close if not forced.
+              # If forced, we will purge the queue and just close
+              rescue ::Rdkafka::RdkafkaError, Rdkafka::AbstractHandle::WaitTimeoutError
+                nil
+              ensure
+                # Purge fully the local queue in case of a forceful shutdown just to be sure, that
+                # there are no dangling messages. In case flush was successful, there should be
+                # none but we do it just in case it timed out
+                purge if force
+              end
+
+              @client.close
+
+              @client = nil
+            end
 
-          # Remove callbacks runners that were registered
-          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
-          ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
-          ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(@id)
+            # Remove callbacks runners that were registered
+            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
+            ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
+            ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(@id)
 
-          @status.closed!
+            @status.closed!
+          end
         end
       end
     end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.8.0'
+  VERSION = '2.8.2'
 end

data/waterdrop.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.cert_chain    = %w[certs/cert.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = []
   spec.require_paths = %w[lib]
 
   spec.metadata = {

metadata

@@ -1,11 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 2.8.2
 platform: ruby
 authors:
 - Maciej Mensfeld
-autorequire: 
 bindir: bin
 cert_chain:
 - |
@@ -35,7 +34,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-09-16 00:00:00.000000000 Z
+date: 2025-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -95,6 +94,8 @@ files:
 - ".coditsu/ci.yml"
 - ".diffend.yml"
 - ".github/FUNDING.yml"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
@@ -105,6 +106,7 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- bin/verify_kafka_warnings
 - certs/cert.pem
 - config/locales/errors.yml
 - docker-compose.yml
@@ -154,7 +156,6 @@ metadata:
   source_code_uri: https://github.com/karafka/waterdrop
   documentation_uri: https://karafka.io/docs/#waterdrop
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -169,8 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Kafka messaging made easy!
 test_files: []