waterdrop 2.4.6 → 2.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a072e6e8ee65e0ce6f5c337c3a56b043140d0a5c60a1fd93d70b329c50932c5
-  data.tar.gz: 3b0d853ed72939da7302a89cca314d105bf3eef7c7abebe0c37c45960c2e8a80
+  metadata.gz: 8aa78b8b5f2d8534689cb9fe3db46d579610ce3b4767cef46b16d8fb1e19d48e
+  data.tar.gz: 75ff38cc56317bc74047fe9ba53ee1f42b098299f694c1ad5767f80ed4c2cf7e
 SHA512:
-  metadata.gz: 50b61ccd6d84ee67a7de95dde8dfc926dd77cc0ebcba6299704a897da88c55a33352a87f0bed725160dd2859e6258d683dc73fec0c92bf3370eeddf1b4490e3c
-  data.tar.gz: a44bdb91da44a8767a385d829eaefd86f214c1baf03b5776c9d3f3d89ac6e2106b07b0da3185208b160110f0cfc5002bb779af66a4682db7d4d96211de7964fd
+  metadata.gz: 3e5589c065d8db716a277bb78e985b85fbe94c2d064eff1cae780f9b4fa6f64ddd95ef0591c99de2dcefedd5582c80ad83442f1cd53091b27349fb193bcd98f6
+  data.tar.gz: 9af7835c4419dd10af2cde93b42c7797e1811f3f049fe7ae4a50a8ccad50fc6f86a8703777c60410d10c1615386eaee2acbeacf073dd640ba24af24ed534326a

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # WaterDrop changelog
 
+## 2.4.7 (2022-12-18)
+- Add support to customizable middlewares that can modify message hash prior to validation and dispatch.
+- Fix a case where upon not-available leader, metadata request would not be retried
+- Require `karafka-core` 2.0.7.
+
 ## 2.4.6 (2022-12-10)
 - Set `statistics.interval.ms` to 5 seconds by default, so the defaults cover all the instrumentation out of the box.
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.6)
-      karafka-core (>= 2.0.6, < 3.0.0)
+    waterdrop (2.4.7)
+      karafka-core (>= 2.0.7, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -22,7 +22,7 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.6)
+    karafka-core (2.0.7)
       concurrent-ruby (>= 1.1)
       rdkafka (>= 0.12)
     mini_portile2 (2.8.0)

data/README.md

@@ -35,6 +35,7 @@ It:
   * [Error notifications](#error-notifications)
   * [Datadog and StatsD integration](#datadog-and-statsd-integration)
   * [Forking and potential memory problems](#forking-and-potential-memory-problems)
+- [Middleware](#middleware)
 - [Note on contributions](#note-on-contributions)
 
 ## Installation
@@ -420,6 +421,38 @@ If you work with forked processes, make sure you **don't** use the producer befo
 
 To tackle this [obstacle](https://github.com/appsignal/rdkafka-ruby/issues/15) related to rdkafka, WaterDrop adds finalizer to each of the producers to close the rdkafka client before the Ruby process is shutdown. Due to the [nature of the finalizers](https://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/), this implementation prevents producers from being GCed (except upon VM shutdown) and can cause memory leaks if you don't use persistent/long-lived producers in a long-running process or if you don't use the `#close` method of a producer when it is no longer needed. Creating a producer instance for each message is anyhow a rather bad idea, so we recommend not to.
 
+## Middleware
+
+WaterDrop supports injecting middleware similar to Rack.
+
+Middleware can be used to provide extra functionalities like auto-serialization of data or any other modifications of messages before their validation and dispatch.
+
+Each middleware accepts the message hash as input and expects a message hash as a result.
+
+There are two methods to register middlewares:
+
+- `#prepend` - registers middleware as the first in the order of execution
+- `#append` - registers middleware as the last in the order of execution
+
+Below you can find an example middleware that converts the incoming payload into a JSON string by running `#to_json` automatically:
+
+```ruby
+class AutoMapper
+  def call(message)
+    message[:payload] = message[:payload].to_json
+    message
+  end
+end
+
+# Register middleware
+producer.middleware.append(AutoMapper.new)
+
+# Dispatch without manual casting
+producer.produce_async(topic: 'users', payload: user)
+```
+
+**Note**: It is up to the end user to decide whether to modify the provided message or deep copy it and update the newly created one.
+
 ## Note on contributions
 
 First, thank you for considering contributing to the Karafka ecosystem! It's people like you that make the open source community such a great community!

data/lib/waterdrop/config.rb

@@ -57,6 +57,12 @@ module WaterDrop
     # rdkafka options
     # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
     setting :kafka, default: {}
+    # Middleware chain that can be expanded with useful middleware steps
+    setting(
+      :middleware,
+      default: false,
+      constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
+    )
 
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module WaterDrop
+  # WaterDrop instrumentation related module
   module Instrumentation
     # Default listener that hooks up to our instrumentation and uses its events for logging
     # It can be removed/replaced or anything without any harm to the Waterdrop flow

data/lib/waterdrop/middleware.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Simple middleware layer for manipulating messages prior to their validation
+  class Middleware
+    def initialize
+      @mutex = Mutex.new
+      @steps = []
+    end
+
+    # Runs middleware on a single message prior to validation
+    #
+    # @param message [Hash] message hash
+    # @return [Hash] message hash. Either the same if transformed in place, or a copy if modified
+    #   into a new object.
+    # @note You need to decide yourself whether you don't use the message hash data anywhere else
+    #   and you want to save on memory by modifying it in place or do you want to do a deep copy
+    def run(message)
+      @steps.each do |step|
+        message = step.call(message)
+      end
+
+      message
+    end
+
+    # @param messages [Array<Hash>] messages on which we want to run middlewares
+    # @return [Array<Hash>] transformed messages
+    def run_many(messages)
+      messages.map do |message|
+        run(message)
+      end
+    end
+
+    # Register given middleware as the first one in the chain
+    # @param step [#call] step that needs to return the message
+    def prepend(step)
+      @mutex.synchronize do
+        @steps.prepend step
+      end
+    end
+
+    # Register given middleware as the last one in the chain
+    # @param step [#call] step that needs to return the message
+    def append(step)
+      @mutex.synchronize do
+        @steps.append step
+      end
+    end
+  end
+end

data/lib/waterdrop/patches/rdkafka/metadata.rb

@@ -7,6 +7,14 @@ module WaterDrop
     module Rdkafka
       # Rdkafka::Metadata patches
       module Metadata
+        # Errors upon which we retry the metadata fetch
+        RETRIED_ERRORS = %i[
+          timed_out
+          leader_not_available
+        ].freeze
+
+        private_constant :RETRIED_ERRORS
+
         # We overwrite this method because there were reports of metadata operation timing out
         # when Kafka was under stress. While the messages dispatch will be retried, metadata
         # fetch happens prior to that, effectively crashing the process. Metadata fetch was not
@@ -19,7 +27,7 @@ module WaterDrop
 
           super(*args)
         rescue ::Rdkafka::RdkafkaError => e
-          raise unless e.code == :timed_out
+          raise unless RETRIED_ERRORS.include?(e.code)
           raise if attempt > 10
 
           backoff_factor = 2**attempt

data/lib/waterdrop/producer/async.rb

@@ -15,6 +15,8 @@ module WaterDrop
       #   message could not be sent to Kafka
       def produce_async(message)
         ensure_active!
+
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -36,6 +38,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def produce_many_async(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/buffer.rb

@@ -19,6 +19,7 @@ module WaterDrop
       #   message could not be sent to Kafka
       def buffer(message)
         ensure_active!
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -37,6 +38,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def buffer_many(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument(

data/lib/waterdrop/producer/sync.rb

@@ -17,6 +17,8 @@ module WaterDrop
       #   message could not be sent to Kafka
       def produce_sync(message)
         ensure_active!
+
+        message = middleware.run(message)
         validate_message!(message)
 
         @monitor.instrument(
@@ -47,6 +49,8 @@ module WaterDrop
       #   and the message could not be sent to Kafka
       def produce_many_sync(messages)
         ensure_active!
+
+        messages = middleware.run_many(messages)
         messages.each { |message| validate_message!(message) }
 
         @monitor.instrument('messages.produced_sync', producer_id: id, messages: messages) do

data/lib/waterdrop/producer.rb

@@ -3,10 +3,13 @@
 module WaterDrop
   # Main WaterDrop messages producer
   class Producer
+    extend Forwardable
     include Sync
     include Async
     include Buffer
 
+    def_delegators :config, :middleware
+
     # @return [String] uuid of the current producer
     attr_reader :id
     # @return [Status] producer status object

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.6'
+  VERSION = '2.4.7'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.6', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.0.7', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7'

data.tar.gz.sig

@@ -1,3 +1,3 @@
-�hlV���8��<��<(ĉr����f�����E\ϟ�}��5�L a�4�M�O�����%�`��Dˣ�M�B�6��P�K��W,a���e��� &ipb
-G�C}�Y��X4��O�Q^�
-#�p;���^�&�U0�q)����YJ7��9�cg����[!� V�򃔑hj�ٌ�TD�d�p�U��p���3Ss�
+��+x��8�XTҞQ-u�8
+q���%P,���+��%�ݗQ ��we�!}qJaP��5j�:H;�#�v�O�F[0�C�ŮP Y�cH�b��=*��
~u+5��N�
+����K�GD��.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.6
+  version: 2.4.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-12-10 00:00:00.000000000 Z
+date: 2022-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: 2.0.7
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -108,6 +108,7 @@ files:
 - lib/waterdrop/instrumentation/notifications.rb
 - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
 - lib/waterdrop/instrumentation/vendors/datadog/listener.rb
+- lib/waterdrop/middleware.rb
 - lib/waterdrop/patches/rdkafka/metadata.rb
 - lib/waterdrop/patches/rdkafka/producer.rb
 - lib/waterdrop/producer.rb