waterdrop 2.7.0.alpha2 → 2.7.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89426650b3e24dded080b6a0ebb1b9db7f23fd289ced679fcba3527f8c06a06b
-  data.tar.gz: 8163dc40307dae99c2409047a237d58bf1b83923d9d2739de3db4aa7839e5c86
+  metadata.gz: d2a4c74fefd80d0de855aa3117c48603d6cbe03351ba5f25f15761d546265dd5
+  data.tar.gz: 7f24353f0a2c48c8b38a1800677b9997bfb9056fdb360245dfaff5d01f625ad0
 SHA512:
-  metadata.gz: 716da4bb23487cf445dd2f43d3b7a6a69046e87d22c39beb5612f2667130efaed2e8a7b5c49a98f85df3c4cc7673065c87023e0d0b90618f869863dc6b015818
-  data.tar.gz: 1c9f6a97e3675570db5f4e2c258e82286d5ec2417ad172f9bc9d1698a2b88564446f2d5c412cee2205b2e91ca9b08144d210a7d0014e7609aa9b38002cff47ae
+  metadata.gz: cada00734dd6c14336368518cab6921b0ed14606950de8c74bb2385ce8621178e9202bff77f73f9196cbcdc6283aa26c3c93487c62818f1d0506e12e2d56b107
+  data.tar.gz: 7885a43d0d2f8e49dcc27d12c186b7f7a4fced13d8ee59c3a521921a7bd7327736d3dd1c54b5c36f01a729a49cf30f321df209e555cfa34e9ab3b96b57caa31c

data/CHANGELOG.md

@@ -4,6 +4,7 @@
 
 This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
 
+- **[Feature]** Support custom OAuth providers.
 - **[Breaking]** Drop Ruby `2.7` support.
 - **[Breaking]** Change default timeouts so final delivery `message.timeout.ms` is less that `max_wait_time` so we do not end up with not final verdict.
 - **[Breaking]** Update all the time related configuration settings to be in `ms` and not mixed.

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.7.0.alpha2)
-      karafka-core (>= 2.4.0.alpha1, < 3.0.0)
+    waterdrop (2.7.0.beta1)
+      karafka-core (>= 2.4.0.beta2, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -31,9 +31,9 @@ GEM
     ffi (1.16.3)
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.0.alpha1)
-      karafka-rdkafka (>= 0.15.0.alpha1, < 0.16.0)
-    karafka-rdkafka (0.15.0.alpha1)
+    karafka-core (2.4.0.beta2)
+      karafka-rdkafka (>= 0.15.0.beta3, < 0.16.0)
+    karafka-rdkafka (0.15.0.beta3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/config/locales/errors.yml

@@ -4,14 +4,20 @@ en:
       missing: must be present
       logger_format: must be present
       deliver_format: must be boolean
+      instrument_on_wait_queue_full_format: must be boolean
       id_format: must be a non-empty string
+      monitor_format: must be present
+      client_class_format: must be present
       max_payload_size_format: must be an integer that is equal or bigger than 1
       max_wait_timeout_format: must be an integer that is equal or bigger than 0
       kafka_format: must be a hash with symbol based keys
       kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       wait_on_queue_full_format: must be boolean
-      wait_backoff_on_queue_full_format: must be a numeric that is bigger or equal to 0
-      wait_timeout_on_queue_full_format: must be a numeric that is bigger or equal to 0
+      wait_backoff_on_queue_full_format: must be a numeric that is equal or bigger to 0
+      wait_timeout_on_queue_full_format: must be a numeric that is equal or bigger to 0
+      wait_backoff_on_transaction_command_format: must be a numeric that is equal or bigger to 0
+      max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
+      oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
 
     message:
       missing: must be present

data/lib/waterdrop/clients/rdkafka.rb

@@ -11,20 +11,50 @@ module WaterDrop
         # @param producer [WaterDrop::Producer] producer instance with its config, etc
         # @note We overwrite this that way, because we do not care
         def new(producer)
-          config = producer.config.kafka.to_h
+          kafka_config = producer.config.kafka.to_h
+          monitor = producer.config.monitor
 
-          client = ::Rdkafka::Config.new(config).producer
+          client = ::Rdkafka::Config.new(kafka_config).producer(native_kafka_auto_start: false)
+
+          # Register statistics runner for this particular type of callbacks
+          ::Karafka::Core::Instrumentation.statistics_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::Statistics.new(producer.id, client.name, monitor)
+          )
+
+          # Register error tracking callback
+          ::Karafka::Core::Instrumentation.error_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::Error.new(producer.id, client.name, monitor)
+          )
+
+          # Register oauth bearer refresh for this particular type of callbacks
+          ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
+            producer.id,
+            Instrumentation::Callbacks::OauthbearerTokenRefresh.new(client, monitor)
+          )
 
           # This callback is not global and is per client, thus we do not have to wrap it with a
           # callbacks manager to make it work
           client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
             producer.id,
             producer.transactional?,
-            producer.config.monitor
+            monitor
           )
 
+          oauth_listener = producer.config.oauth.token_provider_listener
+          # We need to subscribe the oauth listener here because we want it to be ready before
+          # any producer callbacks run. In theory because WaterDrop rdkafka producer is lazy loaded
+          # we would have enough time to make user subscribe it himself, but then it would not
+          # coop with auto-configuration coming from Karafka. The way it is done below, if it is
+          # configured it will be subscribed and if not, user always can subscribe it himself as
+          # long as it is done prior to first usage
+          monitor.subscribe(oauth_listener) if oauth_listener
+
+          client.start
+
           # Switch to the transactional mode if user provided the transactional id
-          client.init_transactions if config.key?(:'transactional.id')
+          client.init_transactions if kafka_config.key?(:'transactional.id')
 
           client
         end

data/lib/waterdrop/config.rb

@@ -13,11 +13,11 @@ module WaterDrop
       # emit librdkafka statistics every five seconds. This is used in instrumentation.
       # When disabled, part of metrics will not be published and available.
       'statistics.interval.ms': 5_000,
-      # We set it to a value that is lower than `max_wait_time` to have a final verdict upon sync
-      # delivery
-      'message.timeout.ms': 55_000,
-      # Must be less or equal to `message.timeout.ms` defaults
-      'transaction.timeout.ms': 45_000
+      # We set it to a value that is lower than `max_wait_timeout` to have a final verdict upon
+      # sync delivery
+      'message.timeout.ms': 50_000,
+      # Must be more or equal to `message.timeout.ms` defaults
+      'transaction.timeout.ms': 55_000
     }.freeze
 
     private_constant :KAFKA_DEFAULTS
@@ -89,6 +89,14 @@ module WaterDrop
       constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
     )
 
+    # Namespace for oauth related configuration
+    setting :oauth do
+      # option [false, #call] Listener for using oauth bearer. This listener will be able to
+      #   get the client name to decide whether to use a single multi-client token refreshing
+      #   or have separate tokens per instance.
+      setting :token_provider_listener, default: false
+    end
+
     # Configuration method
     # @yield Runs a block of code providing a config singleton instance to it
     # @yieldparam [WaterDrop::Config] WaterDrop config instance

data/lib/waterdrop/contracts/config.rb

@@ -14,13 +14,24 @@ module WaterDrop
 
       required(:id) { |val| val.is_a?(String) && !val.empty? }
       required(:logger) { |val| !val.nil? }
+      required(:monitor) { |val| !val.nil? }
       required(:deliver) { |val| [true, false].include?(val) }
       required(:max_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
       required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:client_class) { |val| !val.nil? }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:wait_on_queue_full) { |val| [true, false].include?(val) }
+      required(:instrument_on_wait_queue_full) { |val| [true, false].include?(val) }
       required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
       required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
+      required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
+
+      nested(:oauth) do
+        required(:token_provider_listener) do |val|
+          val == false || val.respond_to?(:on_oauthbearer_token_refresh)
+        end
+      end
 
       # rdkafka allows both symbols and strings as keys for config but then casts them to strings
       # This can be confusing, so we expect all keys to be symbolized

data/lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Instrumentation
+    module Callbacks
+      # Callback that is triggered when oauth token needs to be refreshed.
+      class OauthbearerTokenRefresh
+        # @param bearer [Rdkafka::Producer] given rdkafka instance. It is needed as
+        #   we need to have a reference to call `#oauthbearer_set_token` or
+        #   `#oauthbearer_set_token_failure` upon the event.
+        # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
+        def initialize(bearer, monitor)
+          @bearer = bearer
+          @monitor = monitor
+        end
+
+        # Upon receiving of this event, user is required to invoke either `#oauthbearer_set_token`
+        # or `#oauthbearer_set_token_failure` on the `event[:bearer]` depending whether token
+        # obtaining was successful or not.
+        #
+        # Please refer to WaterDrop and Karafka documentation or `Rdkafka::Helpers::OAuth`
+        # documentation directly for exact parameters of those methods.
+        #
+        # @param _rd_config [Rdkafka::Config]
+        # @param bearer_name [String] name of the bearer for which we refresh
+        def call(_rd_config, bearer_name)
+          return unless @bearer.name == bearer_name
+
+          @monitor.instrument(
+            'oauthbearer.token_refresh',
+            bearer: @bearer,
+            caller: self
+          )
+        end
+      end
+    end
+  end
+end

data/lib/waterdrop/instrumentation/notifications.rb

@@ -21,6 +21,8 @@ module WaterDrop
         messages.produced_sync
         messages.buffered
 
+        oauthbearer.token_refresh
+
         transaction.started
         transaction.committed
         transaction.aborted

data/lib/waterdrop/producer.rb

@@ -104,18 +104,6 @@ module WaterDrop
         @pid = Process.pid
         @client = Builder.new.call(self, @config)
 
-        # Register statistics runner for this particular type of callbacks
-        ::Karafka::Core::Instrumentation.statistics_callbacks.add(
-          @id,
-          Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
-        )
-
-        # Register error tracking callback
-        ::Karafka::Core::Instrumentation.error_callbacks.add(
-          @id,
-          Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
-        )
-
         @status.connected!
         @monitor.instrument('producer.connected', producer_id: id)
       end
@@ -209,6 +197,7 @@ module WaterDrop
           # Remove callbacks runners that were registered
           ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
           ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
+          ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(@id)
 
           @status.closed!
         end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.7.0.alpha2'
+  VERSION = '2.7.0.beta1'
 end

data/waterdrop.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'karafka-core', '>= 2.4.0.alpha1', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.4.0.beta2', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.0.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.7.0.alpha2
+  version: 2.7.0.beta1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-03-17 00:00:00.000000000 Z
+date: 2024-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.alpha1
+        version: 2.4.0.beta2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0.alpha1
+        version: 2.4.0.beta2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -107,6 +107,7 @@ files:
 - lib/waterdrop/helpers/counter.rb
 - lib/waterdrop/instrumentation/callbacks/delivery.rb
 - lib/waterdrop/instrumentation/callbacks/error.rb
+- lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb
 - lib/waterdrop/instrumentation/callbacks/statistics.rb
 - lib/waterdrop/instrumentation/logger_listener.rb
 - lib/waterdrop/instrumentation/monitor.rb