waterdrop 2.4.2 → 2.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba9d9c3c27b55980d16b4bca8266ec70864e53fc00e8f9823bc6407704105638
-  data.tar.gz: '017907f6d33941b52bb8943befe6981586083de52739cf12937ceae3dececd79'
+  metadata.gz: 065c53c0e711cfe49e718a321ce64740b68a45bd6ac477c9eaf2a79876de8df7
+  data.tar.gz: e77179ca5162c4862369838197a62063b5127b4d3a72b7ab08f85f930cee6ea6
 SHA512:
-  metadata.gz: e3d60772d7a278eed00a63ee08690a3897c45d55a74f2dd796353f528f8b6ae0ec93a8f79c8b6c717ad3435f9f3661b811ba278cd35355a2079b0194a6dc56c6
-  data.tar.gz: 721694ab354d11419dc21f2911cbea57f4d89160067ec6b69687741471f498535779e4ee43ece02ac7697a070dfc3d5c6dc2d9119b811fd4b84ecb9703ad89a2
+  metadata.gz: 76af1a5f87a5f31bda8b9f5f111173e1992f6d5ddc85fa2b43d5e1fbd45b93e67601691cbf13b3b2a2e8eb49bc04196b84c5eb81c5b0bd6e11bbe014220fe821
+  data.tar.gz: 27c667d0c93f6948a85d9411fd1dfa46bd38ebb36f7c6907785658bb58ae3001b8d2b81bcb1a9faac64acc5248bc1300a3530736cdd7193b155e01eca4eefbbe

checksums.yaml.gz.sig

@@ -1,3 +1,2 @@
-	��k���H�?	ӻ����Ko�=��O��;z�]͌
-���r��
ȼ	sKZސ��b��M��ۅk�c
-#�)��Ju�LT�õ����/�����S�O	謂!FR׵߃oÈ���ZU��%�.m���̀ٚ[޶�n!���$3@��e7-<P���^�ϩ42��1t�͉�p�&|�&䭠��i��K L��̤���i=W��MS�3o�C4e08g�x����v��B�gU�&W�(��@��A�|�*�X����8q%�Y�I��D��Q]ӮL���,0�#7�
+߸'��A�=�Y撂.�~h��H�ύ̋�`��FI��Bw�+&�����)*�7���
�ʣ5�w� ��3T���)�K����s���9l�PӐ*5�W�>��'J�����}^2YcN׬b��~[�� 9�?�j�5�o�պ�{���2��a�8�
+��,��S�

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/.github/workflows/ci.yml

@@ -23,13 +23,15 @@ jobs:
           - ruby: '3.1'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{matrix.ruby}}
+      - name: Remove libzstd-dev to check no supported compressions
+        run: sudo apt-get -y remove libzstd-dev
       - name: Run Kafka with docker-compose
         # We need to give Kafka enough time to start and create all the needed topics, etc
         # If anyone has a better idea on how to do it smart and easily, please contact me
@@ -54,7 +56,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set up Ruby
@@ -73,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Run Coditsu

data/.ruby-version

@@ -1 +1 @@
-3.1.2
+3.1.3

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # WaterDrop changelog
 
+## 2.4.3 (2022-12-07)
+- Support for librdkafka 0.13
+- Update Github Actions
+- Change auto-generated id from `SecureRandom#uuid` to `SecureRandom#hex(6)`
+- Remove shared components that were moved to `karafka-core` from WaterDrop
+
 ## 2.4.2 (2022-09-29)
 - Allow sending tombstone messages (#267)
 

data/Gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.4.2)
-      karafka-core (>= 2.0.2, < 3.0.0)
-      rdkafka (>= 0.10)
+    waterdrop (2.4.3)
+      karafka-core (>= 2.0.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -23,8 +22,9 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.2)
+    karafka-core (2.0.6)
       concurrent-ruby (>= 1.1)
+      rdkafka (>= 0.12)
     mini_portile2 (2.8.0)
     minitest (5.16.3)
     rake (13.0.6)
@@ -32,19 +32,19 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -53,7 +53,7 @@ GEM
     simplecov_json_formatter (0.1.4)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.0)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   arm64-darwin
@@ -67,4 +67,4 @@ DEPENDENCIES
   waterdrop!
 
 BUNDLED WITH
-   2.3.22
+   2.3.26

data/README.md

@@ -1,11 +1,5 @@
 # WaterDrop
 
-**Note**: Documentation presented here refers to WaterDrop `2.x`.
-
-WaterDrop `2.x` works with Karafka `2.*` and aims to either work as a standalone producer or as a part of the Karafka `2.*`.
-
-Please refer to [this](https://github.com/karafka/waterdrop/tree/1.4) branch and its documentation for details about WaterDrop `1.*` usage.
-
 [![Build Status](https://github.com/karafka/waterdrop/workflows/ci/badge.svg)](https://github.com/karafka/waterdrop/actions?query=workflow%3Aci)
 [![Gem Version](https://badge.fury.io/rb/waterdrop.svg)](http://badge.fury.io/rb/waterdrop)
 [![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
@@ -35,6 +29,7 @@ It:
   * [Buffering](#buffering)
       + [Using WaterDrop to buffer messages based on the application logic](#using-waterdrop-to-buffer-messages-based-on-the-application-logic)
       + [Using WaterDrop with rdkafka buffers to achieve periodic auto-flushing](#using-waterdrop-with-rdkafka-buffers-to-achieve-periodic-auto-flushing)
+  * [Compression](#compression)
 - [Instrumentation](#instrumentation)
   * [Usage statistics](#usage-statistics)
   * [Error notifications](#error-notifications)
@@ -161,9 +156,14 @@ Keep in mind, that message you want to send should be either binary or stringifi
 
 ### Using WaterDrop across the application and with Ruby on Rails
 
-If you plan to both produce and consume messages using Kafka, you should install and use [Karafka](https://github.com/karafka/karafka). It integrates automatically with Ruby on Rails applications and auto-configures WaterDrop producer to make it accessible via `Karafka#producer` method.
+If you plan to both produce and consume messages using Kafka, you should install and use [Karafka](https://github.com/karafka/karafka). It integrates automatically with Ruby on Rails applications and auto-configures WaterDrop producer to make it accessible via `Karafka#producer` method:
+
+```ruby
+event = Events.last
+Karafka.producer.produce_async(topic: 'events', payload: event.to_json)
+```
 
-If you want to only produce messages from within your application, since WaterDrop is thread-safe you can create a single instance in an initializer like so:
+If you want to only produce messages from within your application without consuming with Karafka, since WaterDrop is thread-safe you can create a single instance in an initializer like so:
 
 ```ruby
 KAFKA_PRODUCER = WaterDrop::Producer.new
@@ -200,6 +200,35 @@ WaterDrop producers support buffering messages in their internal buffers and on
 
 This means that depending on your use case, you can achieve both granular buffering and flushing control when needed with context awareness and periodic and size-based flushing functionalities.
 
+### Compression
+
+WaterDrop supports following compression types:
+
+- `gzip`
+- `zstd`
+- `lz4`
+- `snappy`
+
+To use compression, set `kafka` scope `compression.codec` setting. You can also optionally indicate the `compression.level`:
+
+```ruby
+producer = WaterDrop::Producer.new
+
+producer.setup do |config|
+  config.kafka = {
+    'bootstrap.servers': 'localhost:9092',
+    'compression.codec': 'gzip',
+    'compression.level': 6
+  }
+end
+```
+
+**Note**: In order to use `zstd`, you need to install `libzstd-dev`:
+
+```bash
+apt-get install -y libzstd-dev
+```
+
 #### Using WaterDrop to buffer messages based on the application logic
 
 ```ruby

data/config/errors.yml

@@ -17,7 +17,7 @@ en:
       topic_format: 'does not match the topic allowed format'
       partition_key_format: must be a non-empty string
       timestamp_format: must be either time or integer
-      payload_format: must be string
+      payload_format: must be string or nil
       headers_format: must be a hash
       key_format: must be a non-empty string
       payload_max_size: is more than `max_payload_size` config value

data/lib/waterdrop/config.rb

@@ -22,7 +22,7 @@ module WaterDrop
     setting(
       :id,
       default: false,
-      constructor: ->(id) { id || SecureRandom.uuid }
+      constructor: ->(id) { id || SecureRandom.hex(6) }
     )
     # option [Instance] logger that we want to use
     # @note Due to how rdkafka works, this setting is global for all the producers

data/lib/waterdrop/contracts/message.rb

@@ -34,14 +34,14 @@ module WaterDrop
       optional(:timestamp) { |val| val.nil? || (val.is_a?(Time) || val.is_a?(Integer)) }
       optional(:headers) { |val| val.nil? || val.is_a?(Hash) }
 
-      virtual do |config, errors|
+      virtual do |message, errors|
         next true unless errors.empty?
-        next true unless config.key?(:headers)
-        next true if config[:headers].nil?
+        next true unless message.key?(:headers)
+        next true if message[:headers].nil?
 
         errors = []
 
-        config.fetch(:headers).each do |key, value|
+        message.fetch(:headers).each do |key, value|
           errors << [%i[headers], :invalid_key_type] unless key.is_a?(String)
           errors << [%i[headers], :invalid_value_type] unless value.is_a?(String)
         end
@@ -49,10 +49,10 @@ module WaterDrop
         errors
       end
 
-      virtual do |config, errors, validator|
+      virtual do |message, errors, validator|
         next true unless errors.empty?
-        next if config[:payload].nil? # tombstone payload
-        next true if config[:payload].bytesize <= validator.max_payload_size
+        next if message[:payload].nil? # tombstone payload
+        next true if message[:payload].bytesize <= validator.max_payload_size
 
         [[%i[payload], :max_size]]
       end

data/lib/waterdrop/patches/rdkafka/producer.rb

@@ -8,14 +8,22 @@ module WaterDrop
       # Rdkafka::Producer patches
       module Producer
         # Adds a method that allows us to get the native kafka producer name
+        #
+        # In between rdkafka versions, there are internal changes that force us to add some extra
+        # magic to support all the versions.
+        #
         # @return [String] producer instance name
         def name
           unless @_native
             version = ::Gem::Version.new(::Rdkafka::VERSION)
-            change = ::Gem::Version.new('0.12.0')
-            # 0.12.0 changed how the native producer client reference works.
-            # This code supports both older and newer versions of rdkafka
-            @_native = version >= change ? @client.native : @native_kafka
+
+            if version < ::Gem::Version.new('0.12.0')
+              @native = @native_kafka
+            elsif version < ::Gem::Version.new('0.13.0.beta.1')
+              @_native = @client.native
+            else
+              @_native = @native_kafka.inner
+            end
           end
 
           ::Rdkafka::Bindings.rd_kafka_name(@_native)

data/lib/waterdrop/producer.rb

@@ -82,13 +82,13 @@ module WaterDrop
         @client = Builder.new.call(self, @config)
 
         # Register statistics runner for this particular type of callbacks
-        ::WaterDrop::Instrumentation.statistics_callbacks.add(
+        ::Karafka::Core::Instrumentation.statistics_callbacks.add(
           @id,
           Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
         )
 
         # Register error tracking callback
-        ::WaterDrop::Instrumentation.error_callbacks.add(
+        ::Karafka::Core::Instrumentation.error_callbacks.add(
           @id,
           Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
         )
@@ -125,8 +125,8 @@ module WaterDrop
           client.close if @client
 
           # Remove callbacks runners that were registered
-          ::WaterDrop::Instrumentation.statistics_callbacks.delete(@id)
-          ::WaterDrop::Instrumentation.error_callbacks.delete(@id)
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
 
           @status.closed!
         end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.4.2'
+  VERSION = '2.4.3'
 end

data/lib/waterdrop.rb

@@ -3,12 +3,11 @@
 # External components
 # delegate should be removed because we don't need it, we just add it because of ruby-kafka
 %w[
-  karafka-core
   forwardable
-  rdkafka
   json
   zeitwerk
   securerandom
+  karafka-core
 ].each { |lib| require lib }
 
 # WaterDrop library
@@ -27,9 +26,3 @@ loader.inflector.inflect('waterdrop' => 'WaterDrop')
 loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
 loader.setup
 loader.eager_load
-
-# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
-# each callback type. Callback manager allows us to register more than one callback
-# @note Those managers are also used by Karafka for consumer related statistics
-Rdkafka::Config.statistics_callback = WaterDrop::Instrumentation.statistics_callbacks
-Rdkafka::Config.error_callback = WaterDrop::Instrumentation.error_callbacks

data/waterdrop.gemspec

@@ -16,8 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.0.2', '< 3.0.0'
-  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'karafka-core', '>= 2.0.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 2.7'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.4.2
+  version: 2.4.3
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2022-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,24 +53,10 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: 2.0.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: rdkafka
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +80,7 @@ extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
@@ -113,17 +100,14 @@ files:
 - lib/waterdrop/contracts/config.rb
 - lib/waterdrop/contracts/message.rb
 - lib/waterdrop/errors.rb
-- lib/waterdrop/instrumentation.rb
 - lib/waterdrop/instrumentation/callbacks/delivery.rb
 - lib/waterdrop/instrumentation/callbacks/error.rb
 - lib/waterdrop/instrumentation/callbacks/statistics.rb
-- lib/waterdrop/instrumentation/callbacks_manager.rb
 - lib/waterdrop/instrumentation/logger_listener.rb
 - lib/waterdrop/instrumentation/monitor.rb
 - lib/waterdrop/instrumentation/notifications.rb
 - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
 - lib/waterdrop/instrumentation/vendors/datadog/listener.rb
-- lib/waterdrop/patches/rdkafka/bindings.rb
 - lib/waterdrop/patches/rdkafka/producer.rb
 - lib/waterdrop/producer.rb
 - lib/waterdrop/producer/async.rb
@@ -156,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/lib/waterdrop/instrumentation/callbacks_manager.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  module Instrumentation
-    # This manager allows us to register multiple callbacks into a hook that is suppose to support
-    # a single callback
-    class CallbacksManager
-      # @return [::WaterDrop::Instrumentation::CallbacksManager]
-      def initialize
-        @callbacks = Concurrent::Hash.new
-      end
-
-      # Invokes all the callbacks registered one after another
-      #
-      # @param args [Object] any args that should go to the callbacks
-      # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
-      #   callbacks and add new at the same time. Since we don't know when and in what thread
-      #   things are going to be added to the manager, we need to extract values into an array and
-      #   run it. That way we can add new things the same time.
-      def call(*args)
-        @callbacks.values.each { |callback| callback.call(*args) }
-      end
-
-      # Adds a callback to the manager
-      #
-      # @param id [String] id of the callback (used when deleting it)
-      # @param callable [#call] object that responds to a `#call` method
-      def add(id, callable)
-        @callbacks[id] = callable
-      end
-
-      # Removes the callback from the manager
-      # @param id [String] id of the callback we want to remove
-      def delete(id)
-        @callbacks.delete(id)
-      end
-    end
-  end
-end

data/lib/waterdrop/instrumentation.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Namespace for all the things related with WaterDrop instrumentation process
-  module Instrumentation
-    class << self
-      # Builds a manager for statistics callbacks
-      # @return [WaterDrop::CallbacksManager]
-      def statistics_callbacks
-        @statistics_callbacks ||= CallbacksManager.new
-      end
-
-      # Builds a manager for error callbacks
-      # @return [WaterDrop::CallbacksManager]
-      def error_callbacks
-        @error_callbacks ||= CallbacksManager.new
-      end
-    end
-  end
-end

data/lib/waterdrop/patches/rdkafka/bindings.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  module Patches
-    module Rdkafka
-      # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
-      # to work with in a bit different way than rdkafka itself
-      module Bindings
-        class << self
-          # Add extra methods that we need
-          # @param mod [::Rdkafka::Bindings] rdkafka bindings module
-          def included(mod)
-            mod.attach_function :rd_kafka_name, [:pointer], :string
-
-            # Default rdkafka setup for errors doest not propagate client details, thus it always
-            # publishes all the stuff for all rdkafka instances. We change that by providing
-            # function that fetches the instance name, allowing us to have better notifications
-            mod.send(:remove_const, :ErrorCallback)
-            mod.const_set(:ErrorCallback, build_error_callback)
-          end
-
-          # @return [FFI::Function] overwritten callback function
-          def build_error_callback
-            FFI::Function.new(
-              :void, %i[pointer int string pointer]
-            ) do |client_prr, err_code, reason, _opaque|
-              return nil unless ::Rdkafka::Config.error_callback
-
-              name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
-
-              error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
-
-              ::Rdkafka::Config.error_callback.call(name, error)
-            end
-          end
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)