waterdrop 2.4.6 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a072e6e8ee65e0ce6f5c337c3a56b043140d0a5c60a1fd93d70b329c50932c5
-  data.tar.gz: 3b0d853ed72939da7302a89cca314d105bf3eef7c7abebe0c37c45960c2e8a80
+  metadata.gz: 8fab3f2b565a128bcc34f8d8bd10b36b0bd9546d1c52109a97ad87449fcd1731
+  data.tar.gz: 891f381590986451b330f74ac0ab56e6d2873d84a778d33bf470db1fe5e8447d
 SHA512:
-  metadata.gz: 50b61ccd6d84ee67a7de95dde8dfc926dd77cc0ebcba6299704a897da88c55a33352a87f0bed725160dd2859e6258d683dc73fec0c92bf3370eeddf1b4490e3c
-  data.tar.gz: a44bdb91da44a8767a385d829eaefd86f214c1baf03b5776c9d3f3d89ac6e2106b07b0da3185208b160110f0cfc5002bb779af66a4682db7d4d96211de7964fd
+  metadata.gz: de29a31f2d498bfd04577ddd7edec90603d1226bc9b355d3e65e62fae0e2cb0f89bc5708a0734425df0fc5458adcb01e5b59775fda3daaceab1de586b41ed07d
+  data.tar.gz: 052ea95a7c819c42ba62c1275b4895cf9b034e429ef34af0d4556965dd28cd0a9307a0a5613f3a126b4e41da6b0cfe8042e3f614045929203faca67c496d4352

data/.github/workflows/ci.yml

@@ -16,11 +16,12 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.2'
           - '3.1'
           - '3.0'
           - '2.7'
         include:
-          - ruby: '3.1'
+          - ruby: '3.2'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v3
@@ -62,7 +63,7 @@ jobs:
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 3.1
+          ruby-version: 3.2
       - name: Install latest bundler
         run: gem install bundler --no-document
       - name: Install Diffend plugin

data/.ruby-version

@@ -1 +1 @@
-3.1.3
+3.2.0

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # WaterDrop changelog
 
+## 2.4.8 (2023-01-07)
+- Use monotonic time from Karafka core.
+
+## 2.4.7 (2022-12-18)
+- Add support to customizable middlewares that can modify message hash prior to validation and dispatch.
+- Fix a case where upon not-available leader, metadata request would not be retried
+- Require `karafka-core` 2.0.7.
+
 ## 2.4.6 (2022-12-10)
 - Set `statistics.interval.ms` to 5 seconds by default, so the defaults cover all the instrumentation out of the box.
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.6)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    waterdrop (2.4.8)
+      karafka-core (>= 2.0.8, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,11 +22,11 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.6)
+    karafka-core (2.0.8)
       concurrent-ruby (>= 1.1)
       rdkafka (>= 0.12)
-    mini_portile2 (2.8.0)
-    minitest (5.16.3)
+    mini_portile2 (2.8.1)
+    minitest (5.17.0)
     rake (13.0.6)
     rdkafka (0.12.0)
       ffi (~> 1.15)
@@ -38,14 +38,14 @@ GEM
       rspec-mocks (~> 3.12.0)
     rspec-core (3.12.0)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.0)
+    rspec-expectations (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -67,4 +67,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   2.3.26
+   2.4.2

data/README.md

@@ -35,6 +35,7 @@ It:
   * [Error notifications](#error-notifications)
   * [Datadog and StatsD integration](#datadog-and-statsd-integration)
   * [Forking and potential memory problems](#forking-and-potential-memory-problems)
+- [Middleware](#middleware)
 - [Note on contributions](#note-on-contributions)
 
 ## Installation
@@ -420,6 +421,38 @@ If you work with forked processes, make sure you **don't** use the producer befo
 
 To tackle this [obstacle](https://github.com/appsignal/rdkafka-ruby/issues/15) related to rdkafka, WaterDrop adds finalizer to each of the producers to close the rdkafka client before the Ruby process is shutdown. Due to the [nature of the finalizers](https://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/), this implementation prevents producers from being GCed (except upon VM shutdown) and can cause memory leaks if you don't use persistent/long-lived producers in a long-running process or if you don't use the `#close` method of a producer when it is no longer needed. Creating a producer instance for each message is anyhow a rather bad idea, so we recommend not to.
 
+## Middleware
+
+WaterDrop supports injecting middleware similar to Rack.
+
+Middleware can be used to provide extra functionalities like auto-serialization of data or any other modifications of messages before their validation and dispatch.
+
+Each middleware accepts the message hash as input and expects a message hash as a result.
+
+There are two methods to register middlewares:
+
+- `#prepend` - registers middleware as the first in the order of execution
+- `#append` - registers middleware as the last in the order of execution
+
+Below you can find an example middleware that converts the incoming payload into a JSON string by running `#to_json` automatically:
+
+```ruby
+class AutoMapper
+  def call(message)
+    message[:payload] = message[:payload].to_json
+    message
+  end
+end
+
+# Register middleware
+producer.middleware.append(AutoMapper.new)
+
+# Dispatch without manual casting
+producer.produce_async(topic: 'users', payload: user)
+```
+
+**Note**: It is up to the end user to decide whether to modify the provided message or deep copy it and update the newly created one.
+
 ## Note on contributions
 
 First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!

data/lib/waterdrop/config.rb

@@ -57,6 +57,12 @@ module WaterDrop
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}
+    # Middleware chain that can be expanded with useful middleware steps
+    setting(
+      :middleware,
+      default: false,
+      constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
+    )
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it

data/lib/waterdrop/contracts/config.rb

@@ -7,7 +7,7 @@ module WaterDrop
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('config')
       end

data/lib/waterdrop/contracts/message.rb

@@ -8,7 +8,7 @@ module WaterDrop
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+            File.join(WaterDrop.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('message')
       end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module WaterDrop
+  # WaterDrop instrumentation related module
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Waterdrop flow

data/lib/waterdrop/middleware.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Simple middleware layer for manipulating messages prior to their validation
+  class Middleware
+    def initialize
+      @mutex = Mutex.new
+      @steps = []
+    end
+
+    # Runs middleware on a single message prior to validation
+    #
+    # @param message [Hash] message hash
+    # @return [Hash] message hash. Either the same if transformed in place, or a copy if modified
+    #   into a new object.
+    # @note You need to decide yourself whether you don't use the message hash data anywhere else
+    #   and you want to save on memory by modifying it in place or do you want to do a deep copy
+    def run(message)
+      @steps.each do |step|
+        message = step.call(message)
+      end
+
+      message
+    end
+
+    # @param messages [Array<Hash>] messages on which we want to run middlewares
+    # @return [Array<Hash>] transformed messages
+    def run_many(messages)
+      messages.map do |message|
+        run(message)
+      end
+    end
+
+    # Register given middleware as the first one in the chain
+    # @param step [#call] step that needs to return the message
+    def prepend(step)
+      @mutex.synchronize do
+        @steps.prepend step
+      end
+    end
+
+    # Register given middleware as the last one in the chain
+    # @param step [#call] step that needs to return the message
+    def append(step)
+      @mutex.synchronize do
+        @steps.append step
+      end
+    end
+  end
+end

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -7,6 +7,14 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Metadata patches
       module Metadata
+        # Errors upon which we retry the metadata fetch
+        RETRIED_ERRORS = %i[
+          timed_out
+          leader_not_available
+        ].freeze
+
+        private_constant :RETRIED_ERRORS
+
         # We overwrite this method because there were reports of metadata operation timing out
         # when Kafka was under stress. While the messages dispatch will be retried, metadata
         # fetch happens prior to that, effectively crashing the process. Metadata fetch was not
@@ -19,7 +27,7 @@ module WaterDrop
 
           super(*args)
         rescue ::Rdkafka::RdkafkaError => e
-          raise unless e.code == :timed_out
+          raise unless RETRIED_ERRORS.include?(e.code)
           raise if attempt > 10
 
           backoff_factor = 2**attempt

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -7,6 +7,8 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Producer patches
       module Producer
+        include ::Karafka::Core::Helpers::Time
+
         # Cache partitions count for 30 seconds
         PARTITIONS_COUNT_TTL = 30_000
 
@@ -20,7 +22,7 @@ module WaterDrop
             topic_metadata = ::Rdkafka::Metadata.new(inner_kafka, topic).topics&.first
 
             cache[topic] = [
-              now,
+              monotonic_now,
               topic_metadata ? topic_metadata[:partition_count] : nil
             ]
           end
@@ -46,7 +48,7 @@ module WaterDrop
           closed_producer_check(__method__)
 
           @_partitions_count_cache.delete_if do |_, cached|
-            now - cached.first > PARTITIONS_COUNT_TTL
+            monotonic_now - cached.first > PARTITIONS_COUNT_TTL
           end
 
           @_partitions_count_cache[topic].last
@@ -68,11 +70,6 @@ module WaterDrop
 
           @_inner_kafka
         end
-
-        # @return [Float] current clock time
-        def now
-          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) * 1_000
-        end
       end
     end
   end

data/lib/waterdrop/producer/async.rb

@@ -15,6 +15,8 @@ module WaterDrop
       #   message could not be sent to Kafka
       def produce_async(message)
         ensure_active!
+
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -36,6 +38,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def produce_many_async(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/buffer.rb

@@ -19,6 +19,7 @@ module WaterDrop
       #   message could not be sent to Kafka
       def buffer(message)
         ensure_active!
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -37,6 +38,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def buffer_many(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/sync.rb

@@ -17,6 +17,8 @@ module WaterDrop
       #   message could not be sent to Kafka
       def produce_sync(message)
         ensure_active!
+
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -47,6 +49,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do

data/lib/waterdrop/producer.rb

@@ -3,10 +3,13 @@
 module WaterDrop
   # Main WaterDrop messages producer
   class Producer
+    extend Forwardable
     include Sync
     include Async
     include Buffer
 
+    def_delegators :config, :middleware
+
     # @return [String] uuid of the current producer
     attr_reader :id
     # @return [Status] producer status object

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.6'
+  VERSION = '2.4.8'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.6', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.8', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.6
+  version: 2.4.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-10 00:00:00.000000000 Z
+date: 2023-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -92,7 +92,7 @@ files:
 - MIT-LICENSE
 - README.md
 - certs/cert_chain.pem
-- config/errors.yml
+- config/locales/errors.yml
 - docker-compose.yml
 - lib/waterdrop.rb
 - lib/waterdrop/config.rb
@@ -108,6 +108,7 @@ files:
 - lib/waterdrop/instrumentation/notifications.rb
 - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
 - lib/waterdrop/instrumentation/vendors/datadog/listener.rb
+- lib/waterdrop/middleware.rb
 - lib/waterdrop/patches/rdkafka/metadata.rb
 - lib/waterdrop/patches/rdkafka/producer.rb
 - lib/waterdrop/producer.rb
@@ -141,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!