waterdrop 2.7.2 → 2.7.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c580193c1e6a2bdcdcec6e95f99f4c6f7647a1d063871dde565282e9a2988c06
4
- data.tar.gz: 7bc3f33ec9ce355682cea9047d62dfe8614f5319fc30e5a8b7cbd4d43d533182
3
+ metadata.gz: 83e99f22e3b3099cad3e459f171d7d43ab537fd62c50593997c3297846eb124a
4
+ data.tar.gz: 9583295a925fe82e6ca6a6899e0d37f644d0a30cc2a2359289d84ea78ab7e0ec
5
5
  SHA512:
6
- metadata.gz: 8daa029fcc03d3170a1ffe5a17576e5906185391cd28482ecb6c2cf7ad56077e88e0818f87d92228b021dfd9eb22f3f314bce522a267431a7c60b98adf37a668
7
- data.tar.gz: 9c408bc75a9225cf13078abc13b845d8b280f9342626f3a2caaf7216663f2a9da007224f47d29f6bb09a6c4600eb9e69e1965f6b86cb135e7255b1171d649d8a
6
+ metadata.gz: 017dd6f279e40d9c2d338e8ea2763fcab70e3ec99c0f229a1fb5c7037765d8f5a00c0dfe2d2ab932623b6b735ca27d9bd3ca3a8f012dd5aff33952f5eaa2b355
7
+ data.tar.gz: 6ff18fdd690e8e3f2341e2fa90a28b0972421559e5600c2c1c17b77d31d9882ed0476a3f2f7e6c33844f0b2a71aceb0b7f63bd8e6f8689084a41b6cfc09b6525
checksums.yaml.gz.sig CHANGED
Binary file
@@ -18,6 +18,7 @@ jobs:
18
18
  fail-fast: false
19
19
  matrix:
20
20
  ruby:
21
+ - '3.4.0-preview1'
21
22
  - '3.3'
22
23
  - '3.2'
23
24
  - '3.1'
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 3.3.1
1
+ 3.3.2
data/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # WaterDrop changelog
2
2
 
3
+ ## 2.7.3 (2024-06-09)
4
+ - [Enhancement] Introduce `reload_on_transaction_fatal_error` to reload the librdkafka after transactional failures
5
+ - [Enhancement] Flush on fatal transactional errors.
6
+ - [Enhancement] Add topic scope to `report_metric` (YadhuPrakash)
7
+ - [Enhancement] Cache middleware reference saving 1 object allocation on each message dispatch.
8
+ - [Enhancement] Provide `#idempotent?` similar to `#transactional?`.
9
+ - [Enhancement] Provide alias to `#with` named `#variant`.
10
+ - [Fix] Prevent from creating `acks` altering variants on idempotent producers.
11
+
3
12
  ## 2.7.2 (2024-05-09)
4
13
  - [Fix] Fix missing requirement of `delegate` for non-Rails use-cases. Always require delegate for variants usage (samsm)
5
14
 
data/Gemfile CHANGED
@@ -12,6 +12,7 @@ end
12
12
 
13
13
  group :test do
14
14
  gem 'factory_bot'
15
+ gem 'ostruct'
15
16
  gem 'rspec'
16
17
  gem 'simplecov'
17
18
  end
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- waterdrop (2.7.2)
4
+ waterdrop (2.7.3)
5
5
  karafka-core (>= 2.4.0, < 3.0.0)
6
6
  karafka-rdkafka (>= 0.15.1)
7
7
  zeitwerk (~> 2.3)
@@ -9,7 +9,7 @@ PATH
9
9
  GEM
10
10
  remote: https://rubygems.org/
11
11
  specs:
12
- activesupport (7.1.3.2)
12
+ activesupport (7.1.3.3)
13
13
  base64
14
14
  bigdecimal
15
15
  concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -22,7 +22,7 @@ GEM
22
22
  base64 (0.2.0)
23
23
  bigdecimal (3.1.8)
24
24
  byebug (11.1.3)
25
- concurrent-ruby (1.2.3)
25
+ concurrent-ruby (1.3.1)
26
26
  connection_pool (2.4.1)
27
27
  diff-lcs (1.5.1)
28
28
  docile (1.4.0)
@@ -38,9 +38,10 @@ GEM
38
38
  ffi (~> 1.15)
39
39
  mini_portile2 (~> 2.6)
40
40
  rake (> 12)
41
- mini_portile2 (2.8.6)
42
- minitest (5.22.3)
41
+ mini_portile2 (2.8.7)
42
+ minitest (5.23.1)
43
43
  mutex_m (0.2.0)
44
+ ostruct (0.6.0)
44
45
  rake (13.2.1)
45
46
  rspec (3.13.0)
46
47
  rspec-core (~> 3.13.0)
@@ -63,7 +64,7 @@ GEM
63
64
  simplecov_json_formatter (0.1.4)
64
65
  tzinfo (2.0.6)
65
66
  concurrent-ruby (~> 1.0)
66
- zeitwerk (2.6.13)
67
+ zeitwerk (2.6.15)
67
68
 
68
69
  PLATFORMS
69
70
  ruby
@@ -72,9 +73,10 @@ PLATFORMS
72
73
  DEPENDENCIES
73
74
  byebug
74
75
  factory_bot
76
+ ostruct
75
77
  rspec
76
78
  simplecov
77
79
  waterdrop!
78
80
 
79
81
  BUNDLED WITH
80
- 2.5.9
82
+ 2.5.11
@@ -17,6 +17,7 @@ en:
17
17
  wait_timeout_on_queue_full_format: must be a numeric that is equal or bigger to 0
18
18
  wait_backoff_on_transaction_command_format: must be a numeric that is equal or bigger to 0
19
19
  max_attempts_on_transaction_command_format: must be an integer that is equal or bigger than 1
20
+ reload_on_transaction_fatal_error_format: must be boolean
20
21
  oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
21
22
 
22
23
  variant:
@@ -25,7 +26,7 @@ en:
25
26
  max_wait_timeout_format: must be an integer that is equal or bigger than 0
26
27
  kafka_key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
27
28
  kafka_key_not_per_topic: This config option cannot be set on a per topic basis
28
- kafka_key_acks_not_changeable: Acks value cannot be redefined for a transactional producer
29
+ kafka_key_acks_not_changeable: Acks value cannot be redefined for a transactional or idempotent producer
29
30
 
30
31
  message:
31
32
  missing: must be present
@@ -72,6 +72,10 @@ module WaterDrop
72
72
  # option [Numeric] How many times to retry a retryable transaction related error before
73
73
  # giving up
74
74
  setting :max_attempts_on_transaction_command, default: 5
75
+ # When a fatal transactional error occurs, should we close and recreate the underlying producer
76
+ # to keep going or should we stop. Since we will open a new instance and the failed transaction
77
+ # anyhow rolls back, we should be able to safely reload.
78
+ setting :reload_on_transaction_fatal_error, default: true
75
79
 
76
80
  # option [Boolean] should we send messages. Setting this to false can be really useful when
77
81
  # testing and or developing because when set to false, won't actually ping Kafka but will
@@ -26,6 +26,7 @@ module WaterDrop
26
26
  required(:wait_timeout_on_queue_full) { |val| val.is_a?(Numeric) && val >= 0 }
27
27
  required(:wait_backoff_on_transaction_command) { |val| val.is_a?(Numeric) && val >= 0 }
28
28
  required(:max_attempts_on_transaction_command) { |val| val.is_a?(Integer) && val >= 1 }
29
+ required(:reload_on_transaction_fatal_error) { |val| [true, false].include?(val) }
29
30
 
30
31
  nested(:oauth) do
31
32
  required(:token_provider_listener) do |val|
@@ -85,6 +85,23 @@ module WaterDrop
85
85
 
86
86
  errors
87
87
  end
88
+
89
+ # Prevent from creating variants altering acks when idempotent
90
+ virtual do |config, errors|
91
+ next true unless errors.empty?
92
+ # Relevant only for the transactional producer
93
+ next true unless config.fetch(:idempotent)
94
+
95
+ errors = []
96
+
97
+ config
98
+ .fetch(:topic_config)
99
+ .keys
100
+ .select { |key| key.to_s.include?('acks') }
101
+ .each { |key| errors << [[:kafka, key], :kafka_key_acks_not_changeable] }
102
+
103
+ errors
104
+ end
88
105
  end
89
106
  end
90
107
  end
@@ -129,6 +129,11 @@ module WaterDrop
129
129
  info(event, 'Closing producer')
130
130
  end
131
131
 
132
+ # @param event [Dry::Events::Event] event that happened with the details
133
+ def on_producer_reloaded(event)
134
+ info(event, 'Producer successfully reloaded')
135
+ end
136
+
132
137
  # @param event [Dry::Events::Event] event that happened with the error details
133
138
  def on_error_occurred(event)
134
139
  error = event[:error]
@@ -10,6 +10,7 @@ module WaterDrop
10
10
  producer.connected
11
11
  producer.closing
12
12
  producer.closed
13
+ producer.reloaded
13
14
 
14
15
  message.produced_async
15
16
  message.produced_sync
@@ -199,6 +199,15 @@ module WaterDrop
199
199
  tags: default_tags + ["broker:#{broker_statistics['nodename']}"]
200
200
  )
201
201
  end
202
+ when :topics
203
+ statistics.fetch('topics').each_value do |topic_statistics|
204
+ public_send(
205
+ metric.type,
206
+ metric.name,
207
+ topic_statistics.dig(*metric.key_location),
208
+ tags: default_tags + ["topic:#{topic_statistics['topic']}"]
209
+ )
210
+ end
202
211
  else
203
212
  raise ArgumentError, metric.scope
204
213
  end
@@ -4,10 +4,16 @@ module WaterDrop
4
4
  class Producer
5
5
  # Transactions related producer functionalities
6
6
  module Transactions
7
+ # We should never reload producer if it was fenced, otherwise we could end up with some sort
8
+ # of weird race-conditions
9
+ NON_RELOADABLE_ERRORS = %i[
10
+ fenced
11
+ ].freeze
12
+
7
13
  # Contract to validate that input for transactional offset storage is correct
8
14
  CONTRACT = Contracts::TransactionalOffset.new
9
15
 
10
- private_constant :CONTRACT
16
+ private_constant :CONTRACT, :NON_RELOADABLE_ERRORS
11
17
 
12
18
  # Creates a transaction.
13
19
  #
@@ -79,11 +85,27 @@ module WaterDrop
79
85
  #
80
86
  # rubocop:disable Lint/RescueException
81
87
  rescue Exception => e
82
- # rubocop:enable Lint/RescueException
83
- with_transactional_error_handling(:abort) do
84
- transactional_instrument(:aborted) { client.abort_transaction }
88
+ # This code is a bit tricky. We have an error and when it happens we try to rollback
89
+ # the transaction. However we may end up in a state where transaction aborting itself
90
+ # produces error. In such case we also want to handle it as fatal and reload client.
91
+ # This is why we catch this here
92
+ begin
93
+ # rubocop:enable Lint/RescueException
94
+ with_transactional_error_handling(:abort) do
95
+ transactional_instrument(:aborted) do
96
+ client.abort_transaction
97
+ end
98
+ end
99
+ rescue StandardError => e
100
+ # If something from rdkafka leaks here, it means there was a non-retryable error that
101
+ # bubbled up. In such cases if we should, we do reload the underling client
102
+ transactional_reload_client_if_needed(e)
103
+
104
+ raise
85
105
  end
86
106
 
107
+ transactional_reload_client_if_needed(e)
108
+
87
109
  raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
88
110
  end
89
111
  end
@@ -197,7 +219,12 @@ module WaterDrop
197
219
  attempt: attempt
198
220
  )
199
221
 
200
- raise if e.fatal?
222
+ if e.fatal?
223
+ # Reload the client on fatal errors if requested
224
+ transactional_reload_client_if_needed(e)
225
+
226
+ raise
227
+ end
201
228
 
202
229
  if do_retry
203
230
  # Backoff more and more before retries
@@ -212,12 +239,43 @@ module WaterDrop
212
239
  with_transactional_error_handling(:abort, allow_abortable: false) do
213
240
  transactional_instrument(:aborted) { client.abort_transaction }
214
241
  end
215
-
216
- raise
217
242
  end
218
243
 
219
244
  raise
220
245
  end
246
+
247
+ # Reloads the underlying client instance if needed and allowed
248
+ #
249
+ # This should be used only in transactions as only then we can get fatal transactional
250
+ # errors and we can safely reload the client.
251
+ #
252
+ # @param error [Exception] any error that was raised
253
+ #
254
+ # @note We only reload on rdkafka errors that are a cause on messages dispatches.
255
+ # Because we reload on any errors where cause is `Rdkafka::RdkafkaError` (minus exclusions)
256
+ # this in theory can cause reload if it was something else that raised those in transactions,
257
+ # for example Karafka. This is a trade-off. Since any error anyhow will cause a rollback,
258
+ # putting aside performance implication of closing and reconnecting, this should not be an
259
+ # issue.
260
+ def transactional_reload_client_if_needed(error)
261
+ rd_error = error.is_a?(Rdkafka::RdkafkaError) ? error : error.cause
262
+
263
+ return unless rd_error.is_a?(Rdkafka::RdkafkaError)
264
+ return unless config.reload_on_transaction_fatal_error
265
+ return if NON_RELOADABLE_ERRORS.include?(rd_error.code)
266
+
267
+ @operating_mutex.synchronize do
268
+ @monitor.instrument(
269
+ 'producer.reloaded',
270
+ producer_id: id
271
+ ) do
272
+ @client.flush(current_variant.max_wait_timeout)
273
+ purge
274
+ @client.close
275
+ @client = Builder.new.call(self, @config)
276
+ end
277
+ end
278
+ end
221
279
  end
222
280
  end
223
281
  end
@@ -98,7 +98,9 @@ module WaterDrop
98
98
  # We pass this to validation, to make sure no-one alters the `acks` value when operating
99
99
  # in the transactional mode as it causes librdkafka to crash ruby
100
100
  # @see https://github.com/confluentinc/librdkafka/issues/4710
101
- transactional: @producer.transactional?
101
+ transactional: @producer.transactional?,
102
+ # We pass this for a similar reason as above
103
+ idempotent: @producer.idempotent?
102
104
  }
103
105
  end
104
106
  end
@@ -21,7 +21,7 @@ module WaterDrop
21
21
 
22
22
  private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH
23
23
 
24
- def_delegators :config, :middleware
24
+ def_delegators :config
25
25
 
26
26
  # @return [String] uuid of the current producer
27
27
  attr_reader :id
@@ -143,6 +143,33 @@ module WaterDrop
143
143
  end
144
144
  end
145
145
 
146
+ # Builds the variant alteration and returns it.
147
+ #
148
+ # @param args [Object] anything `Producer::Variant` initializer accepts
149
+ # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
150
+ def with(**args)
151
+ ensure_active!
152
+
153
+ Variant.new(self, **args)
154
+ end
155
+
156
+ alias variant with
157
+
158
+ # @return [Boolean] true if current producer is idempotent
159
+ def idempotent?
160
+ # Every transactional producer is idempotent by default always
161
+ return true if transactional?
162
+ return @idempotent if instance_variable_defined?(:'@idempotent')
163
+
164
+ @idempotent = config.kafka.to_h.key?(:'enable.idempotence')
165
+ end
166
+
167
+ # Returns and caches the middleware object that may be used
168
+ # @return [WaterDrop::Producer::Middleware]
169
+ def middleware
170
+ @middleware ||= config.middleware
171
+ end
172
+
146
173
  # Flushes the buffers in a sync way and closes the producer
147
174
  # @param force [Boolean] should we force closing even with outstanding messages after the
148
175
  # max wait timeout
@@ -210,16 +237,6 @@ module WaterDrop
210
237
  end
211
238
  end
212
239
 
213
- # Builds the variant alteration and returns it.
214
- #
215
- # @param args [Object] anything `Producer::Variant` initializer accepts
216
- # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
217
- def with(**args)
218
- ensure_active!
219
-
220
- Variant.new(self, **args)
221
- end
222
-
223
240
  # Closes the producer with forced close after timeout, purging any outgoing data
224
241
  def close!
225
242
  close(force: true)
@@ -306,7 +323,7 @@ module WaterDrop
306
323
  # in an infinite loop, effectively hanging the processing
307
324
  raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full
308
325
 
309
- label = caller_locations(2, 1)[0].label.split(' ').last
326
+ label = caller_locations(2, 1)[0].label.split(' ').last.split('#').last
310
327
 
311
328
  # We use this syntax here because we want to preserve the original `#cause` when we
312
329
  # instrument the error and there is no way to manually assign `#cause` value. We want to keep
@@ -3,5 +3,5 @@
3
3
  # WaterDrop library
4
4
  module WaterDrop
5
5
  # Current WaterDrop version
6
- VERSION = '2.7.2'
6
+ VERSION = '2.7.3'
7
7
  end
data.tar.gz.sig CHANGED
Binary file
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.7.2
4
+ version: 2.7.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
35
35
  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
36
  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
37
37
  -----END CERTIFICATE-----
38
- date: 2024-05-09 00:00:00.000000000 Z
38
+ date: 2024-06-09 00:00:00.000000000 Z
39
39
  dependencies:
40
40
  - !ruby/object:Gem::Dependency
41
41
  name: karafka-core
metadata.gz.sig CHANGED
Binary file