waterdrop 2.7.0.alpha2 → 2.7.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 89426650b3e24dded080b6a0ebb1b9db7f23fd289ced679fcba3527f8c06a06b
4
- data.tar.gz: 8163dc40307dae99c2409047a237d58bf1b83923d9d2739de3db4aa7839e5c86
3
+ metadata.gz: d2a4c74fefd80d0de855aa3117c48603d6cbe03351ba5f25f15761d546265dd5
4
+ data.tar.gz: 7f24353f0a2c48c8b38a1800677b9997bfb9056fdb360245dfaff5d01f625ad0
5
5
  SHA512:
6
- metadata.gz: 716da4bb23487cf445dd2f43d3b7a6a69046e87d22c39beb5612f2667130efaed2e8a7b5c49a98f85df3c4cc7673065c87023e0d0b90618f869863dc6b015818
7
- data.tar.gz: 1c9f6a97e3675570db5f4e2c258e82286d5ec2417ad172f9bc9d1698a2b88564446f2d5c412cee2205b2e91ca9b08144d210a7d0014e7609aa9b38002cff47ae
6
+ metadata.gz: cada00734dd6c14336368518cab6921b0ed14606950de8c74bb2385ce8621178e9202bff77f73f9196cbcdc6283aa26c3c93487c62818f1d0506e12e2d56b107
7
+ data.tar.gz: 7885a43d0d2f8e49dcc27d12c186b7f7a4fced13d8ee59c3a521921a7bd7327736d3dd1c54b5c36f01a729a49cf30f321df209e555cfa34e9ab3b96b57caa31c
checksums.yaml.gz.sig CHANGED
Binary file
data/CHANGELOG.md CHANGED
@@ -4,6 +4,7 @@
4
4
 
5
5
  This release contains **BREAKING** changes. Make sure to read and apply upgrade notes.
6
6
 
7
+ - **[Feature]** Support custom OAuth providers.
7
8
  - **[Breaking]** Drop Ruby `2.7` support.
8
9
  - **[Breaking]** Change default timeouts so final delivery `message.timeout.ms` is less that `max_wait_time` so we do not end up with not final verdict.
9
10
  - **[Breaking]** Update all the time related configuration settings to be in `ms` and not mixed.
data/Gemfile.lock CHANGED
@@ -1,8 +1,8 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- waterdrop (2.7.0.alpha2)
5
- karafka-core (>= 2.4.0.alpha1, < 3.0.0)
4
+ waterdrop (2.7.0.beta1)
5
+ karafka-core (>= 2.4.0.beta2, < 3.0.0)
6
6
  zeitwerk (~> 2.3)
7
7
 
8
8
  GEM
@@ -31,9 +31,9 @@ GEM
31
31
  ffi (1.16.3)
32
32
  i18n (1.14.4)
33
33
  concurrent-ruby (~> 1.0)
34
- karafka-core (2.4.0.alpha1)
35
- karafka-rdkafka (>= 0.15.0.alpha1, < 0.16.0)
36
- karafka-rdkafka (0.15.0.alpha1)
34
+ karafka-core (2.4.0.beta2)
35
+ karafka-rdkafka (>= 0.15.0.beta3, < 0.16.0)
36
+ karafka-rdkafka (0.15.0.beta3)
37
37
  ffi (~> 1.15)
38
38
  mini_portile2 (~> 2.6)
39
39
  rake (> 12)
@@ -4,14 +4,20 @@ en:
4
4
  missing: must be present
5
5
  logger_format: must be present
6
6
  deliver_format: must be boolean
7
+ instrument_on_wait_queue_full_format: must be boolean
7
8
  id_format: must be a non-empty string
9
+ monitor_format: must be present
10
+ client_class_format: must be present
8
11
  max_payload_size_format: must be an integer that is equal or bigger than 1
9
12
  max_wait_timeout_format: must be an integer that is equal or bigger than 0
10
13
  kafka_format: must be a hash with symbol based keys
11
14
  kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
12
15
  wait_on_queue_full_format: must be boolean
13
- wait_backoff_on_queue_full_format: must be a numeric that is bigger or equal to 0
14
- wait_timeout_on_queue_full_format: must be a numeric that is bigger or equal to 0
16
+ wait_backoff_on_queue_full_format: must be a numeric that is equal or bigger to 0
17
+ wait_timeout_on_queue_full_format: must be a numeric that is equal or bigger to 0
18
+ wait_backoff_on_transaction_command_format: must be a numeric that is equal or bigger to 0
19
+ max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
20
+ oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
15
21
 
16
22
  message:
17
23
  missing: must be present
@@ -11,20 +11,50 @@ module WaterDrop
11
11
  # @param producer [WaterDrop::Producer] producer instance with its config, etc
12
12
  # @note We overwrite this that way, because we do not care
13
13
  def new(producer)
14
- config = producer.config.kafka.to_h
14
+ kafka_config = producer.config.kafka.to_h
15
+ monitor = producer.config.monitor
15
16
 
16
- client = ::Rdkafka::Config.new(config).producer
17
+ client = ::Rdkafka::Config.new(kafka_config).producer(native_kafka_auto_start: false)
18
+
19
+ # Register statistics runner for this particular type of callbacks
20
+ ::Karafka::Core::Instrumentation.statistics_callbacks.add(
21
+ producer.id,
22
+ Instrumentation::Callbacks::Statistics.new(producer.id, client.name, monitor)
23
+ )
24
+
25
+ # Register error tracking callback
26
+ ::Karafka::Core::Instrumentation.error_callbacks.add(
27
+ producer.id,
28
+ Instrumentation::Callbacks::Error.new(producer.id, client.name, monitor)
29
+ )
30
+
31
+ # Register oauth bearer refresh for this particular type of callbacks
32
+ ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
33
+ producer.id,
34
+ Instrumentation::Callbacks::OauthbearerTokenRefresh.new(client, monitor)
35
+ )
17
36
 
18
37
  # This callback is not global and is per client, thus we do not have to wrap it with a
19
38
  # callbacks manager to make it work
20
39
  client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
21
40
  producer.id,
22
41
  producer.transactional?,
23
- producer.config.monitor
42
+ monitor
24
43
  )
25
44
 
45
+ oauth_listener = producer.config.oauth.token_provider_listener
46
+ # We need to subscribe the oauth listener here because we want it to be ready before
47
+ # any producer callbacks run. In theory because WaterDrop rdkafka producer is lazy loaded
48
+ # we would have enough time to make user subscribe it himself, but then it would not
49
+ # coop with auto-configuration coming from Karafka. The way it is done below, if it is
50
+ # configured it will be subscribed and if not, user always can subscribe it himself as
51
+ # long as it is done prior to first usage
52
+ monitor.subscribe(oauth_listener) if oauth_listener
53
+
54
+ client.start
55
+
26
56
  # Switch to the transactional mode if user provided the transactional id
27
- client.init_transactions if config.key?(:'transactional.id')
57
+ client.init_transactions if kafka_config.key?(:'transactional.id')
28
58
 
29
59
  client
30
60
  end
@@ -13,11 +13,11 @@ module WaterDrop
13
13
  # emit librdkafka statistics every five seconds. This is used in instrumentation.
14
14
  # When disabled, part of metrics will not be published and available.
15
15
  'statistics.interval.ms': 5_000,
16
- # We set it to a value that is lower than `max_wait_time` to have a final verdict upon sync
17
- # delivery
18
- 'message.timeout.ms': 55_000,
19
- # Must be less or equal to `message.timeout.ms` defaults
20
- 'transaction.timeout.ms': 45_000
16
+ # We set it to a value that is lower than `max_wait_timeout` to have a final verdict upon
17
+ # sync delivery
18
+ 'message.timeout.ms': 50_000,
19
+ # Must be more or equal to `message.timeout.ms` defaults
20
+ 'transaction.timeout.ms': 55_000
21
21
  }.freeze
22
22
 
23
23
  private_constant :KAFKA_DEFAULTS
@@ -89,6 +89,14 @@ module WaterDrop
89
89
  constructor: ->(middleware) { middleware || WaterDrop::Middleware.new }
90
90
  )
91
91
 
92
+ # Namespace for oauth related configuration
93
+ setting :oauth do
94
+ # option [false, #call] Listener for using oauth bearer. This listener will be able to
95
+ # get the client name to decide whether to use a single multi-client token refreshing
96
+ # or have separate tokens per instance.
97
+ setting :token_provider_listener, default: false
98
+ end
99
+
92
100
  # Configuration method
93
101
  # @yield Runs a block of code providing a config singleton instance to it
94
102
  # @yieldparam [WaterDrop::Config] WaterDrop config instance
@@ -14,13 +14,24 @@ module WaterDrop
14
14
 
15
15
  required(:id) { |val| val.is_a?(String) && !val.empty? }
16
16
  required(:logger) { |val| !val.nil? }
17
+ required(:monitor) { |val| !val.nil? }
17
18
  required(:deliver) { |val| [true, false].include?(val) }
18
19
  required(:max_payload_size) { |val| val.is_a?(Integer) && val >= 1 }
19
20
  required(:max_wait_timeout) { |val| val.is_a?(Numeric) && val >= 0 }
21
+ required(:client_class) { |val| !val.nil? }
20
22
  required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
21
23
  required(:wait_on_queue_full) { |val| [true, false].include?(val) }
24
+ required(:instrument_on_wait_queue_full) { |val| [true, false].include?(val) }
22
25
  required(:wait_backoff_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
23
26
  required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
27
+ required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
28
+ required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
29
+
30
+ nested(:oauth) do
31
+ required(:token_provider_listener) do |val|
32
+ val == false || val.respond_to?(:on_oauthbearer_token_refresh)
33
+ end
34
+ end
24
35
 
25
36
  # rdkafka allows both symbols and strings as keys for config but then casts them to strings
26
37
  # This can be confusing, so we expect all keys to be symbolized
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WaterDrop
4
+ module Instrumentation
5
+ module Callbacks
6
+ # Callback that is triggered when oauth token needs to be refreshed.
7
+ class OauthbearerTokenRefresh
8
+ # @param bearer [Rdkafka::Producer] given rdkafka instance. It is needed as
9
+ # we need to have a reference to call `#oauthbearer_set_token` or
10
+ # `#oauthbearer_set_token_failure` upon the event.
11
+ # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
12
+ def initialize(bearer, monitor)
13
+ @bearer = bearer
14
+ @monitor = monitor
15
+ end
16
+
17
+ # Upon receiving of this event, user is required to invoke either `#oauthbearer_set_token`
18
+ # or `#oauthbearer_set_token_failure` on the `event[:bearer]` depending whether token
19
+ # obtaining was successful or not.
20
+ #
21
+ # Please refer to WaterDrop and Karafka documentation or `Rdkafka::Helpers::OAuth`
22
+ # documentation directly for exact parameters of those methods.
23
+ #
24
+ # @param _rd_config [Rdkafka::Config]
25
+ # @param bearer_name [String] name of the bearer for which we refresh
26
+ def call(_rd_config, bearer_name)
27
+ return unless @bearer.name == bearer_name
28
+
29
+ @monitor.instrument(
30
+ 'oauthbearer.token_refresh',
31
+ bearer: @bearer,
32
+ caller: self
33
+ )
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -21,6 +21,8 @@ module WaterDrop
21
21
  messages.produced_sync
22
22
  messages.buffered
23
23
 
24
+ oauthbearer.token_refresh
25
+
24
26
  transaction.started
25
27
  transaction.committed
26
28
  transaction.aborted
@@ -104,18 +104,6 @@ module WaterDrop
104
104
  @pid = Process.pid
105
105
  @client = Builder.new.call(self, @config)
106
106
 
107
- # Register statistics runner for this particular type of callbacks
108
- ::Karafka::Core::Instrumentation.statistics_callbacks.add(
109
- @id,
110
- Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
111
- )
112
-
113
- # Register error tracking callback
114
- ::Karafka::Core::Instrumentation.error_callbacks.add(
115
- @id,
116
- Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
117
- )
118
-
119
107
  @status.connected!
120
108
  @monitor.instrument('producer.connected', producer_id: id)
121
109
  end
@@ -209,6 +197,7 @@ module WaterDrop
209
197
  # Remove callbacks runners that were registered
210
198
  ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
211
199
  ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
200
+ ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(@id)
212
201
 
213
202
  @status.closed!
214
203
  end
@@ -3,5 +3,5 @@
3
3
  # WaterDrop library
4
4
  module WaterDrop
5
5
  # Current WaterDrop version
6
- VERSION = '2.7.0.alpha2'
6
+ VERSION = '2.7.0.beta1'
7
7
  end
data/waterdrop.gemspec CHANGED
@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
16
16
  spec.description = spec.summary
17
17
  spec.license = 'MIT'
18
18
 
19
- spec.add_dependency 'karafka-core', '>= 2.4.0.alpha1', '< 3.0.0'
19
+ spec.add_dependency 'karafka-core', '>= 2.4.0.beta2', '< 3.0.0'
20
20
  spec.add_dependency 'zeitwerk', '~> 2.3'
21
21
 
22
22
  spec.required_ruby_version = '>= 3.0.0'
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.7.0.alpha2
4
+ version: 2.7.0.beta1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
36
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
37
  -----END CERTIFICATE-----
38
- date: 2024-03-17 00:00:00.000000000 Z
38
+ date: 2024-03-26 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
43
43
  requirements:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
- version: 2.4.0.alpha1
46
+ version: 2.4.0.beta2
47
47
  - - "<"
48
48
  - !ruby/object:Gem::Version
49
49
  version: 3.0.0
@@ -53,7 +53,7 @@ dependencies:
53
53
  requirements:
54
54
  - - ">="
55
55
  - !ruby/object:Gem::Version
56
- version: 2.4.0.alpha1
56
+ version: 2.4.0.beta2
57
57
  - - "<"
58
58
  - !ruby/object:Gem::Version
59
59
  version: 3.0.0
@@ -107,6 +107,7 @@ files:
107
107
  - lib/waterdrop/helpers/counter.rb
108
108
  - lib/waterdrop/instrumentation/callbacks/delivery.rb
109
109
  - lib/waterdrop/instrumentation/callbacks/error.rb
110
+ - lib/waterdrop/instrumentation/callbacks/oauthbearer_token_refresh.rb
110
111
  - lib/waterdrop/instrumentation/callbacks/statistics.rb
111
112
  - lib/waterdrop/instrumentation/logger_listener.rb
112
113
  - lib/waterdrop/instrumentation/monitor.rb
metadata.gz.sig CHANGED
Binary file