waterdrop 2.8.12 → 2.8.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a0890fcd73147d293340b55cca58c975d81726208c609be1f338ce56d87b4d18
4
- data.tar.gz: 7a2743c068af9936186fa852d6e663c85c172aa93de76b16aaf09e2c1de24a25
3
+ metadata.gz: 57d1e899a312c47c5101a9029914468881e0ebb9437926767078c37f1ce1b4d1
4
+ data.tar.gz: 1767c8a557324bb5bccf93d79847705ecdbe1587d5ff97210597c41ee9cfa0d4
5
5
  SHA512:
6
- metadata.gz: 66136311cb010a1648a0a5440cd930fcb96bb75fe30575d5c14effb8550a7cea4295dda98828790ac567ec9f3346dde7901c11228f6ca251fdf454a5354fad9d
7
- data.tar.gz: a701a63fe3a171084b63ac46ca9f66d75572b7efbfed0b794884df0fdb2b272404a6594ba239f16956cdc9c5fbecb31dbb5574fc0e5c1f12225ce61514c776a3
6
+ metadata.gz: 5286d33cd2bf81f947ddb8c08c77b580aba8ff818f1e01431df36ae31dfbaf7984d3720e4463e1e9551379630fcfc6bcff161e5502d4b9bfa583c913ca050f7c
7
+ data.tar.gz: 7365a54e15b397d1c108db6949c6048c3c7e125b2063f2483e13c4ed6d38f961faaeed101a280cb5b9ef4c8d21335fb429e764c5939dbcf558eebc35aa21f3db
@@ -17,7 +17,6 @@ jobs:
17
17
  specs:
18
18
  timeout-minutes: 15
19
19
  runs-on: ubuntu-latest
20
- needs: diffend
21
20
  env:
22
21
  BUNDLE_FORCE_RUBY_PLATFORM: ${{ matrix.force_ruby_platform }}
23
22
  strategy:
@@ -87,29 +86,6 @@ jobs:
87
86
  - name: Check test topics naming convention
88
87
  run: bin/verify_topics_naming
89
88
 
90
- diffend:
91
- timeout-minutes: 5
92
-
93
- runs-on: ubuntu-latest
94
- strategy:
95
- fail-fast: false
96
- steps:
97
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
98
- with:
99
- fetch-depth: 0
100
- - name: Set up Ruby
101
- uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
102
- with:
103
- ruby-version: 3.4
104
- self-hosted: false
105
-
106
- - name: Install latest bundler
107
- run: gem install bundler --no-document
108
- - name: Install Diffend plugin
109
- run: bundle plugin install diffend
110
- - name: Bundle Secure
111
- run: bundle secure
112
-
113
89
  coditsu:
114
90
  timeout-minutes: 5
115
91
  runs-on: ubuntu-latest
@@ -134,14 +110,29 @@ jobs:
134
110
  - name: Run Coditsu
135
111
  run: ./coditsu_script.sh
136
112
 
113
+ yard-lint:
114
+ timeout-minutes: 5
115
+ runs-on: ubuntu-latest
116
+ steps:
117
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
118
+ with:
119
+ fetch-depth: 0
120
+ - name: Set up Ruby
121
+ uses: ruby/setup-ruby@2a7b30092b0caf9c046252510f9273b4875f3db9 # v1.254.0
122
+ with:
123
+ ruby-version: '3.4'
124
+ bundler-cache: true
125
+ - name: Run yard-lint
126
+ run: bundle exec yard-lint lib/
127
+
137
128
  ci-success:
138
129
  name: CI Success
139
130
  runs-on: ubuntu-latest
140
131
  if: always()
141
132
  needs:
142
- - diffend
143
133
  - coditsu
144
134
  - specs
135
+ - yard-lint
145
136
  steps:
146
137
  - name: Check all jobs passed
147
138
  if: |
data/.yard-lint.yml ADDED
@@ -0,0 +1,174 @@
1
+ # YARD-Lint Configuration
2
+ # See https://github.com/mensfeld/yard-lint for documentation
3
+
4
+ # Global settings for all validators
5
+ AllValidators:
6
+ # YARD command-line options (applied to all validators by default)
7
+ YardOptions:
8
+ - --private
9
+ - --protected
10
+
11
+ # Global file exclusion patterns
12
+ Exclude:
13
+ - '\.git'
14
+ - 'vendor/**/*'
15
+ - 'node_modules/**/*'
16
+ - 'spec/**/*'
17
+ - 'test/**/*'
18
+
19
+ # Exit code behavior (error, warning, convention, never)
20
+ FailOnSeverity: error
21
+
22
+ # Minimum documentation coverage percentage (0-100)
23
+ # Fails if coverage is below this threshold
24
+ MinCoverage: 100
25
+
26
+ # Diff mode settings
27
+ DiffMode:
28
+ # Default base ref for --diff (auto-detects main/master if not specified)
29
+ DefaultBaseRef: ~
30
+
31
+ # Documentation validators
32
+ Documentation/UndocumentedObjects:
33
+ Description: 'Checks for classes, modules, and methods without documentation.'
34
+ Enabled: true
35
+ Severity: error
36
+ ExcludedMethods:
37
+ - 'initialize/0' # Exclude parameter-less initialize
38
+ - '/^_/' # Exclude private methods (by convention)
39
+
40
+ Documentation/UndocumentedMethodArguments:
41
+ Description: 'Checks for method parameters without @param tags.'
42
+ Enabled: true
43
+ Severity: error
44
+
45
+ Documentation/UndocumentedBooleanMethods:
46
+ Description: 'Checks that question mark methods document their boolean return.'
47
+ Enabled: true
48
+ Severity: error
49
+
50
+ Documentation/UndocumentedOptions:
51
+ Description: 'Detects methods with options hash parameters but no @option tags.'
52
+ Enabled: true
53
+ Severity: error
54
+
55
+ Documentation/MarkdownSyntax:
56
+ Description: 'Detects common markdown syntax errors in documentation.'
57
+ Enabled: true
58
+ Severity: error
59
+
60
+ # Tags validators
61
+ Tags/Order:
62
+ Description: 'Enforces consistent ordering of YARD tags.'
63
+ Enabled: true
64
+ Severity: error
65
+ EnforcedOrder:
66
+ - param
67
+ - option
68
+ - return
69
+ - raise
70
+ - example
71
+
72
+ Tags/InvalidTypes:
73
+ Description: 'Validates type definitions in @param, @return, @option tags.'
74
+ Enabled: true
75
+ Severity: error
76
+ ValidatedTags:
77
+ - param
78
+ - option
79
+ - return
80
+
81
+ Tags/TypeSyntax:
82
+ Description: 'Validates YARD type syntax using YARD parser.'
83
+ Enabled: true
84
+ Severity: error
85
+ ValidatedTags:
86
+ - param
87
+ - option
88
+ - return
89
+ - yieldreturn
90
+
91
+ Tags/MeaninglessTag:
92
+ Description: 'Detects @param/@option tags on classes, modules, or constants.'
93
+ Enabled: true
94
+ Severity: error
95
+ CheckedTags:
96
+ - param
97
+ - option
98
+ InvalidObjectTypes:
99
+ - class
100
+ - module
101
+ - constant
102
+
103
+ Tags/CollectionType:
104
+ Description: 'Validates Hash collection syntax consistency.'
105
+ Enabled: true
106
+ Severity: error
107
+ EnforcedStyle: long # 'long' for Hash{K => V} (YARD standard), 'short' for {K => V}
108
+ ValidatedTags:
109
+ - param
110
+ - option
111
+ - return
112
+ - yieldreturn
113
+
114
+ Tags/TagTypePosition:
115
+ Description: 'Validates type annotation position in tags.'
116
+ Enabled: true
117
+ Severity: error
118
+ CheckedTags:
119
+ - param
120
+ - option
121
+ # EnforcedStyle: 'type_after_name' (YARD standard: @param name [Type])
122
+ # or 'type_first' (@param [Type] name)
123
+ EnforcedStyle: type_after_name
124
+
125
+ Tags/ApiTags:
126
+ Description: 'Enforces @api tags on public objects.'
127
+ Enabled: false # Opt-in validator
128
+ Severity: error
129
+ AllowedApis:
130
+ - public
131
+ - private
132
+ - internal
133
+
134
+ Tags/OptionTags:
135
+ Description: 'Requires @option tags for methods with options parameters.'
136
+ Enabled: true
137
+ Severity: error
138
+
139
+ # Warnings validators - catches YARD parser errors
140
+ Warnings/UnknownTag:
141
+ Description: 'Detects unknown YARD tags.'
142
+ Enabled: true
143
+ Severity: error
144
+
145
+ Warnings/UnknownDirective:
146
+ Description: 'Detects unknown YARD directives.'
147
+ Enabled: true
148
+ Severity: error
149
+
150
+ Warnings/InvalidTagFormat:
151
+ Description: 'Detects malformed tag syntax.'
152
+ Enabled: true
153
+ Severity: error
154
+
155
+ Warnings/InvalidDirectiveFormat:
156
+ Description: 'Detects malformed directive syntax.'
157
+ Enabled: true
158
+ Severity: error
159
+
160
+ Warnings/DuplicatedParameterName:
161
+ Description: 'Detects duplicate @param tags.'
162
+ Enabled: true
163
+ Severity: error
164
+
165
+ Warnings/UnknownParameterName:
166
+ Description: 'Detects @param tags for non-existent parameters.'
167
+ Enabled: true
168
+ Severity: error
169
+
170
+ # Semantic validators
171
+ Semantic/AbstractMethods:
172
+ Description: 'Ensures @abstract methods do not have real implementations.'
173
+ Enabled: true
174
+ Severity: error
data/CHANGELOG.md CHANGED
@@ -1,5 +1,18 @@
1
1
  # WaterDrop changelog
2
2
 
3
+ ## 2.8.14 (2025-11-14)
4
+ - [Fix] Fatal error replacement not working without exact error unwind from the fatal envelope
5
+ - [Testing] Add `WaterDrop::Producer::Testing` module for injecting and querying librdkafka fatal errors in tests.
6
+ - [Testing] Add `#trigger_test_fatal_error(error_code, reason)` method to simulate fatal errors without requiring actual broker-side conditions.
7
+ - [Testing] Add `#fatal_error` method to query current fatal error state for validation in tests.
8
+ - [Testing] Add comprehensive test coverage for fatal error injection, reload behavior, and event instrumentation with real librdkafka errors.
9
+ - [Change] Require `karafka-rdkafka` `>=` `0.23.1` due to error handling fixes.
10
+
11
+ ## 2.8.13 (2025-10-31)
12
+ - [Enhancement] Make `fenced` error skip-reload behavior configurable via new `non_reloadable_errors` setting (defaults to `[:fenced]` for backward compatibility).
13
+ - [Enhancement] Add `producer.reload` event allowing config modification before reload to escape fencing loops (#706).
14
+ - [Enhancement] Do not early initialize the new instance on reload.
15
+
3
16
  ## 2.8.12 (2025-10-10)
4
17
  - [Enhancement] Introduce `reload_on_idempotent_fatal_error` to automatically reload librdkafka producer after fatal errors on idempotent (non-transactional) producers.
5
18
  - [Enhancement] Add configurable backoff and retry limits for fatal error recovery to prevent infinite reload loops:
@@ -19,7 +32,7 @@
19
32
  ## 2.8.11 (2025-09-27)
20
33
  - [Enhancement] Provide fast-track for middleware-less flows (20% faster) for single message, 5000x faster for batches.
21
34
  - [Enhancement] Optimize middlewares application by around 20%.
22
- - [Change] Remove Ruby `3.1` according to the EOL schedule.
35
+ - **[EOL]** Remove Ruby `3.1` according to the EOL schedule.
23
36
  - [Fix] Connection pool timeout parameter now accepts milliseconds instead of seconds for consistency with other WaterDrop timeouts. The default timeout has been changed from `5` seconds to `5000` milliseconds (equivalent value).
24
37
 
25
38
  ## 2.8.10 (2025-09-25)
@@ -33,7 +46,7 @@
33
46
  ## 2.8.8 (2025-09-23)
34
47
  - [Feature] Add `WaterDrop::ConnectionPool` for efficient connection pooling using the proven `connection_pool` gem.
35
48
  - [Feature] Add `WaterDrop.instrumentation` class-level instrumentation for producer lifecycle events. This allows external libraries to subscribe to `producer.created` and `producer.configured` events without needing producer instance references, enabling middleware injection and configuration by libraries like Datadog tracing.
36
- - [Change] Remove Ruby `3.1` specs according to the EOL schedule.
49
+ - **[EOL]** Remove Ruby `3.1` specs according to the EOL schedule.
37
50
 
38
51
  ## 2.8.7 (2025-09-02)
39
52
  - [Enhancement] Disable Nagle algorithm by default (improves latency / aligned with librdkafka)
data/Gemfile CHANGED
@@ -2,12 +2,10 @@
2
2
 
3
3
  source 'https://rubygems.org'
4
4
 
5
- plugin 'diffend'
6
-
7
5
  gemspec
8
6
 
9
7
  # Relaxed from 2.7 because we support Ruby 3.1
10
- gem 'zeitwerk', '~> 2.6.18'
8
+ gem 'zeitwerk', '~> 2.7.0'
11
9
 
12
10
  group :development do
13
11
  gem 'byebug'
@@ -19,4 +17,5 @@ group :test do
19
17
  gem 'rspec'
20
18
  gem 'simplecov'
21
19
  gem 'warning'
20
+ gem 'yard-lint'
22
21
  end
data/Gemfile.lock CHANGED
@@ -1,9 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- waterdrop (2.8.12)
4
+ waterdrop (2.8.14)
5
5
  karafka-core (>= 2.4.9, < 3.0.0)
6
- karafka-rdkafka (>= 0.20.0)
6
+ karafka-rdkafka (>= 0.23.1)
7
7
  zeitwerk (~> 2.3)
8
8
 
9
9
  GEM
@@ -24,36 +24,36 @@ GEM
24
24
  ffi (1.17.2-x86_64-darwin)
25
25
  ffi (1.17.2-x86_64-linux-gnu)
26
26
  ffi (1.17.2-x86_64-linux-musl)
27
- json (2.13.2)
28
- karafka-core (2.5.6)
27
+ json (2.15.1)
28
+ karafka-core (2.5.7)
29
29
  karafka-rdkafka (>= 0.20.0)
30
30
  logger (>= 1.6.0)
31
- karafka-rdkafka (0.21.0)
32
- ffi (~> 1.15)
31
+ karafka-rdkafka (0.23.1)
32
+ ffi (~> 1.17.1)
33
33
  json (> 2.0)
34
34
  logger
35
35
  mini_portile2 (~> 2.6)
36
36
  rake (> 12)
37
- karafka-rdkafka (0.21.0-aarch64-linux-gnu)
38
- ffi (~> 1.15)
37
+ karafka-rdkafka (0.23.1-aarch64-linux-gnu)
38
+ ffi (~> 1.17.1)
39
39
  json (> 2.0)
40
40
  logger
41
41
  mini_portile2 (~> 2.6)
42
42
  rake (> 12)
43
- karafka-rdkafka (0.21.0-arm64-darwin)
44
- ffi (~> 1.15)
43
+ karafka-rdkafka (0.23.1-arm64-darwin)
44
+ ffi (~> 1.17.1)
45
45
  json (> 2.0)
46
46
  logger
47
47
  mini_portile2 (~> 2.6)
48
48
  rake (> 12)
49
- karafka-rdkafka (0.21.0-x86_64-linux-gnu)
50
- ffi (~> 1.15)
49
+ karafka-rdkafka (0.23.1-x86_64-linux-gnu)
50
+ ffi (~> 1.17.1)
51
51
  json (> 2.0)
52
52
  logger
53
53
  mini_portile2 (~> 2.6)
54
54
  rake (> 12)
55
- karafka-rdkafka (0.21.0-x86_64-linux-musl)
56
- ffi (~> 1.15)
55
+ karafka-rdkafka (0.23.1-x86_64-linux-musl)
56
+ ffi (~> 1.17.1)
57
57
  json (> 2.0)
58
58
  logger
59
59
  mini_portile2 (~> 2.6)
@@ -62,27 +62,31 @@ GEM
62
62
  mini_portile2 (2.8.9)
63
63
  ostruct (0.6.3)
64
64
  rake (13.3.0)
65
- rspec (3.13.1)
65
+ rspec (3.13.2)
66
66
  rspec-core (~> 3.13.0)
67
67
  rspec-expectations (~> 3.13.0)
68
68
  rspec-mocks (~> 3.13.0)
69
- rspec-core (3.13.4)
69
+ rspec-core (3.13.6)
70
70
  rspec-support (~> 3.13.0)
71
71
  rspec-expectations (3.13.5)
72
72
  diff-lcs (>= 1.2.0, < 2.0)
73
73
  rspec-support (~> 3.13.0)
74
- rspec-mocks (3.13.5)
74
+ rspec-mocks (3.13.6)
75
75
  diff-lcs (>= 1.2.0, < 2.0)
76
76
  rspec-support (~> 3.13.0)
77
- rspec-support (3.13.4)
77
+ rspec-support (3.13.6)
78
78
  simplecov (0.22.0)
79
79
  docile (~> 1.1)
80
80
  simplecov-html (~> 0.11)
81
81
  simplecov_json_formatter (~> 0.1)
82
- simplecov-html (0.13.1)
82
+ simplecov-html (0.13.2)
83
83
  simplecov_json_formatter (0.1.4)
84
84
  warning (1.5.0)
85
- zeitwerk (2.6.18)
85
+ yard (0.9.37)
86
+ yard-lint (1.2.3)
87
+ yard (~> 0.9)
88
+ zeitwerk (~> 2.6)
89
+ zeitwerk (2.7.3)
86
90
 
87
91
  PLATFORMS
88
92
  aarch64-linux-gnu
@@ -105,7 +109,8 @@ DEPENDENCIES
105
109
  simplecov
106
110
  warning
107
111
  waterdrop!
108
- zeitwerk (~> 2.6.18)
112
+ yard-lint
113
+ zeitwerk (~> 2.7.0)
109
114
 
110
115
  BUNDLED WITH
111
116
  2.7.0
@@ -23,6 +23,7 @@ en:
23
23
  reload_on_transaction_fatal_error_format: must be boolean
24
24
  wait_backoff_on_transaction_fatal_error_format: must be a numeric that is equal or bigger to 0
25
25
  max_attempts_on_transaction_fatal_error_format: must be an integer that is equal or bigger than 1
26
+ non_reloadable_errors_format: must be an array of symbols
26
27
  oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
27
28
  idle_disconnect_timeout_format: 'must be an integer that is equal to 0 or bigger than 30 000 (30 seconds)'
28
29
 
@@ -49,6 +49,9 @@ module WaterDrop
49
49
  # @param topic [String, Symbol] topic where we want to dispatch message
50
50
  # @param partition [Integer] target partition
51
51
  # @param _args [Hash] remaining details that are ignored in the dummy mode
52
+ # @option _args [String] :payload message payload
53
+ # @option _args [String, nil] :key message key
54
+ # @option _args [Hash, nil] :headers message headers
52
55
  # @return [Handle] delivery handle
53
56
  def produce(topic:, partition: 0, **_args)
54
57
  Handle.new(topic.to_s, partition, @counters["#{topic}#{partition}"] += 1)
@@ -95,6 +95,20 @@ module WaterDrop
95
95
  # option [Integer] How many times to attempt reloading on transactional fatal error before
96
96
  # giving up. This prevents infinite reload loops if the producer never recovers.
97
97
  setting :max_attempts_on_transaction_fatal_error, default: 10
98
+ # option [Array<Symbol>] List of fatal error codes that should NOT trigger producer reload.
99
+ # These errors represent states that cannot be recovered by simply recreating the client.
100
+ #
101
+ # WARNING: Modifying this setting can cause infinite reload loops if not properly understood.
102
+ # The default includes :fenced errors because:
103
+ # - Fencing occurs when another producer with the same transactional.id takes over
104
+ # - This is an unrecoverable state - reloading won't help as the other producer is active
105
+ # - Attempting to reload on fenced errors creates: produce -> fenced -> reload -> produce ->
106
+ # fenced -> reload (infinite loop)
107
+ #
108
+ # Only remove :fenced from this list if you have explicit logic to handle producer fencing
109
+ # in your application (e.g., coordinated transactional.id assignment, manual intervention).
110
+ # In most cases, you should keep the default value.
111
+ setting :non_reloadable_errors, default: %i[fenced]
98
112
  # option [Integer] Idle disconnect timeout in milliseconds. When set to 0, idle disconnection
99
113
  # is disabled. When set to a positive value, WaterDrop will automatically disconnect
100
114
  # producers that haven't sent any messages for the specified time period. This helps preserve
@@ -30,6 +30,9 @@ module WaterDrop
30
30
  required(:max_attempts_on_idempotent_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
31
31
  required(:wait_backoff_on_transaction_fatal_error) { |val| val.is_a?(Numeric) && val >= 0 }
32
32
  required(:max_attempts_on_transaction_fatal_error) { |val| val.is_a?(Integer) && val >= 1 }
33
+ required(:non_reloadable_errors) do |val|
34
+ val.is_a?(Array) && val.all?(Symbol)
35
+ end
33
36
  required(:idle_disconnect_timeout) do |val|
34
37
  val.is_a?(Integer) && (val.zero? || val >= 30_000)
35
38
  end
@@ -8,6 +8,7 @@ module WaterDrop
8
8
  # @return [Integer] current value
9
9
  attr_reader :value
10
10
 
11
+ # Creates a new counter initialized to 0
11
12
  def initialize
12
13
  @value = 0
13
14
  @mutex = Mutex.new
@@ -22,7 +22,7 @@ module WaterDrop
22
22
 
23
23
  private_constant :RD_KAFKA_RESP_PURGE_QUEUE, :RD_KAFKA_RESP_PURGE_INFLIGHT, :PURGE_ERRORS
24
24
 
25
- # @param producer_id [String] id of the current producer
25
+ # @param producer_id [String]
26
26
  # @param transactional [Boolean] is this handle for a transactional or regular producer
27
27
  # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
28
28
  def initialize(producer_id, transactional, monitor)
@@ -5,7 +5,7 @@ module WaterDrop
5
5
  module Callbacks
6
6
  # Callback that kicks in when error occurs and is published in a background thread
7
7
  class Error
8
- # @param producer_id [String] id of the current producer
8
+ # @param producer_id [String]
9
9
  # @param client_name [String] rdkafka client name
10
10
  # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
11
11
  def initialize(producer_id, client_name, monitor)
@@ -10,7 +10,7 @@ module WaterDrop
10
10
  # previous statistics emit but from the beginning of the process. We decorate it with diff
11
11
  # of all the numeric values against the data from the previous callback emit
12
12
  class Statistics
13
- # @param producer_id [String] id of the current producer
13
+ # @param producer_id [String]
14
14
  # @param client_name [String] rdkafka client name
15
15
  # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
16
16
  def initialize(producer_id, client_name, monitor)
@@ -10,6 +10,7 @@ module WaterDrop
10
10
  producer.connected
11
11
  producer.closing
12
12
  producer.closed
13
+ producer.reload
13
14
  producer.reloaded
14
15
  producer.disconnecting
15
16
  producer.disconnected
@@ -3,6 +3,7 @@
3
3
  module WaterDrop
4
4
  # Simple middleware layer for manipulating messages prior to their validation
5
5
  class Middleware
6
+ # Creates a new middleware chain with no registered steps
6
7
  def initialize
7
8
  @mutex = Mutex.new
8
9
  @steps = []
@@ -23,13 +23,13 @@ module WaterDrop
23
23
  # - Producer is idempotent
24
24
  # - Producer is not transactional
25
25
  # - reload_on_idempotent_fatal_error config is enabled
26
- # - Error is not in the NON_RELOADABLE_FATAL_ERRORS list
26
+ # - Error is not in the non_reloadable_errors config list
27
27
  def idempotent_reloadable?(error)
28
28
  return false unless error.fatal?
29
29
  return false unless idempotent?
30
30
  return false if transactional?
31
31
  return false unless config.reload_on_idempotent_fatal_error
32
- return false if NON_RELOADABLE_FATAL_ERRORS.include?(error.code)
32
+ return false if config.non_reloadable_errors.include?(error.code)
33
33
 
34
34
  true
35
35
  end
@@ -50,21 +50,34 @@ module WaterDrop
50
50
  # old client, and create a new client instance to continue operations.
51
51
  #
52
52
  # @param attempt [Integer] the current reload attempt number
53
+ # @param error [Rdkafka::RdkafkaError] the error that triggered the reload
53
54
  #
54
55
  # @note This is only called for idempotent, non-transactional producers when
55
56
  # `reload_on_idempotent_fatal_error` is enabled
56
57
  # @note After reload, the producer will automatically retry the failed operation
57
- def idempotent_reload_client_on_fatal_error(attempt)
58
+ def idempotent_reload_client_on_fatal_error(attempt, error)
58
59
  @operating_mutex.synchronize do
60
+ # Emit producer.reload event before reload
61
+ # Users can subscribe to this event and modify event[:caller].config.kafka to change
62
+ # producer config
63
+ @monitor.instrument(
64
+ 'producer.reload',
65
+ producer_id: id,
66
+ error: error,
67
+ attempt: attempt,
68
+ caller: self
69
+ )
70
+
71
+ # Clear cached state that depends on config
72
+ # We always clear @idempotent as it might have been modified via the event
73
+ @idempotent = nil
74
+
59
75
  @monitor.instrument(
60
76
  'producer.reloaded',
61
77
  producer_id: id,
62
78
  attempt: attempt
63
79
  ) do
64
- @client.flush(current_variant.max_wait_timeout)
65
- purge
66
- @client.close
67
- @client = Builder.new.call(self, @config)
80
+ reload!
68
81
  end
69
82
  end
70
83
  end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WaterDrop
4
+ class Producer
5
+ # Testing utilities for WaterDrop Producer instances.
6
+ #
7
+ # This module provides methods for triggering and querying fatal errors on producers,
8
+ # which is useful for testing error handling and recovery logic (such as automatic
9
+ # producer reloading on fatal errors).
10
+ #
11
+ # @note This module should only be used in test environments.
12
+ # @note Requires karafka-rdkafka >= 0.23.1 which includes Rdkafka::Testing support.
13
+ # @note This module is not auto-loaded by Zeitwerk and must be manually required.
14
+ #
15
+ # @example Including for a single producer instance
16
+ # require 'waterdrop/producer/testing'
17
+ #
18
+ # producer = WaterDrop::Producer.new
19
+ # producer.singleton_class.include(WaterDrop::Producer::Testing)
20
+ # producer.trigger_test_fatal_error(47, "Test producer fencing")
21
+ #
22
+ # @example Including for all producers in a test suite
23
+ # # In spec_helper.rb or test setup:
24
+ # require 'waterdrop/producer/testing'
25
+ #
26
+ # WaterDrop::Producer.include(WaterDrop::Producer::Testing)
27
+ #
28
+ # @example Testing idempotent producer reload on fatal error
29
+ # producer = WaterDrop::Producer.new do |config|
30
+ # config.kafka = { 'bootstrap.servers': 'localhost:9092' }
31
+ # config.reload_on_idempotent_fatal_error = true
32
+ # end
33
+ # producer.singleton_class.include(WaterDrop::Producer::Testing)
34
+ #
35
+ # # Trigger a fatal error that should cause reload
36
+ # producer.trigger_test_fatal_error(47, "Invalid producer epoch")
37
+ #
38
+ # # Produce should succeed after automatic reload
39
+ # producer.produce_sync(topic: 'test', payload: 'message')
40
+ #
41
+ # # Fatal error should be cleared after reload
42
+ # expect(producer.fatal_error).to be_nil
43
+ module Testing
44
+ # Triggers a test fatal error on the underlying rdkafka producer.
45
+ #
46
+ # This method uses librdkafka's test error injection functionality to simulate
47
+ # fatal errors without requiring actual error conditions. This is particularly
48
+ # useful for testing WaterDrop's fatal error handling and automatic reload logic.
49
+ #
50
+ # @param error_code [Integer] The librdkafka error code to trigger.
51
+ # Common error codes for testing:
52
+ # - 47 (RD_KAFKA_RESP_ERR_INVALID_PRODUCER_EPOCH) - Producer fencing
53
+ # - 64 (RD_KAFKA_RESP_ERR_INVALID_PRODUCER_ID_MAPPING) - Invalid producer ID
54
+ # @param reason [String] A descriptive reason for the error, used for debugging
55
+ # and logging purposes
56
+ #
57
+ # @return [Integer] Result code from rd_kafka_test_fatal_error (0 on success)
58
+ #
59
+ # @raise [RuntimeError] If the underlying rdkafka client doesn't support testing
60
+ #
61
+ # @example Trigger producer fencing error
62
+ # producer.trigger_test_fatal_error(47, "Test producer fencing scenario")
63
+ #
64
+ # @example Trigger invalid producer ID error
65
+ # producer.trigger_test_fatal_error(64, "Test invalid producer ID mapping")
66
+ def trigger_test_fatal_error(error_code, reason)
67
+ ensure_testing_support!
68
+ client.trigger_test_fatal_error(error_code, reason)
69
+ end
70
+
71
+ # Checks if a fatal error has occurred on the underlying rdkafka producer.
72
+ #
73
+ # This method queries librdkafka's fatal error state to retrieve information
74
+ # about any fatal error that has occurred. Fatal errors are serious errors that
75
+ # prevent the producer from continuing normal operation.
76
+ #
77
+ # @return [Hash, nil] A hash containing error details if a fatal error occurred,
78
+ # or nil if no fatal error is present. The hash contains:
79
+ # - :error_code [Integer] The librdkafka error code
80
+ # - :error_string [String] Human-readable error description
81
+ #
82
+ # @example Check for fatal error
83
+ # if error = producer.fatal_error
84
+ # puts "Fatal error #{error[:error_code]}: #{error[:error_string]}"
85
+ # else
86
+ # puts "No fatal error present"
87
+ # end
88
+ #
89
+ # @example Verify fatal error after triggering
90
+ # producer.trigger_test_fatal_error(47, "Test error")
91
+ # error = producer.fatal_error
92
+ # expect(error[:error_code]).to eq(47)
93
+ def fatal_error
94
+ ensure_testing_support!
95
+ client.fatal_error
96
+ end
97
+
98
+ private
99
+
100
+ # Ensures the underlying rdkafka client has testing support available.
101
+ # Automatically requires and includes Rdkafka::Testing if not already present.
102
+ #
103
+ # @return [void]
104
+ # @api private
105
+ def ensure_testing_support!
106
+ return if client.respond_to?(:trigger_test_fatal_error)
107
+
108
+ # Require the rdkafka testing module if not already loaded
109
+ require 'rdkafka/producer/testing' unless defined?(::Rdkafka::Testing)
110
+
111
+ client.singleton_class.include(::Rdkafka::Testing)
112
+ end
113
+ end
114
+ end
115
+ end
@@ -281,24 +281,41 @@ module WaterDrop
281
281
 
282
282
  return unless rd_error.is_a?(Rdkafka::RdkafkaError)
283
283
  return unless config.reload_on_transaction_fatal_error
284
- return if NON_RELOADABLE_FATAL_ERRORS.include?(rd_error.code)
284
+ return if config.non_reloadable_errors.include?(rd_error.code)
285
285
 
286
286
  # Check if we've exceeded max reload attempts
287
287
  return unless transactional_retryable?
288
+ # We bubble up transactional errors, so there are cases where when fencing is not
289
+ # considered a non-reloadable, two layers of error handling would attempt to reload the
290
+ # client causing double reload. This halts reload if we're in a configured state as it
291
+ # means, we've already reloaded and we are not even yet connected
292
+ return if @status.configured?
288
293
 
289
294
  # Increment attempts before reload
290
295
  @transaction_fatal_error_attempts += 1
291
296
 
292
297
  @operating_mutex.synchronize do
298
+ # Emit producer.reload event before reload
299
+ # Users can subscribe to this event and modify event[:caller].config.kafka to change
300
+ # producer config. This is useful for escaping fencing loops by changing transactional.id
301
+ @monitor.instrument(
302
+ 'producer.reload',
303
+ producer_id: id,
304
+ error: rd_error,
305
+ attempt: @transaction_fatal_error_attempts,
306
+ caller: self
307
+ )
308
+
309
+ # Clear cached state that depends on config
310
+ # We always clear @transactional as it might have been modified via the event
311
+ @transactional = nil
312
+
293
313
  @monitor.instrument(
294
314
  'producer.reloaded',
295
315
  producer_id: id,
296
316
  attempt: @transaction_fatal_error_attempts
297
317
  ) do
298
- @client.flush(current_variant.max_wait_timeout)
299
- purge
300
- @client.close
301
- @client = Builder.new.call(self, @config)
318
+ reload!
302
319
  end
303
320
  end
304
321
 
@@ -22,12 +22,6 @@ module WaterDrop
22
22
  Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
23
23
  ].freeze
24
24
 
25
- # We should never reload producer on certain fatal errors as they may indicate state that
26
- # cannot be recovered by simply recreating the client
27
- NON_RELOADABLE_FATAL_ERRORS = %i[
28
- fenced
29
- ].freeze
30
-
31
25
  # Empty hash to save on memory allocations
32
26
  EMPTY_HASH = {}.freeze
33
27
 
@@ -35,7 +29,7 @@ module WaterDrop
35
29
  EMPTY_ARRAY = [].freeze
36
30
 
37
31
  private_constant(
38
- :SUPPORTED_FLOW_ERRORS, :NON_RELOADABLE_FATAL_ERRORS, :EMPTY_HASH, :EMPTY_ARRAY
32
+ :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH, :EMPTY_ARRAY
39
33
  )
40
34
 
41
35
  def_delegators :config
@@ -211,8 +205,13 @@ module WaterDrop
211
205
 
212
206
  # Builds the variant alteration and returns it.
213
207
  #
214
- # @param args [Object] anything `Producer::Variant` initializer accepts
208
+ # @param args [Hash] variant configuration options
209
+ # @option args [Integer, nil] :max_wait_timeout alteration to max wait timeout or nil to use
210
+ # default
211
+ # @option args [Hash] :topic_config extra topic configuration that can be altered
212
+ # @option args [Boolean] :default is this a default variant or an altered one
215
213
  # @return [WaterDrop::Producer::Variant] variant proxy to use with alterations
214
+ # @see https://karafka.io/docs/Librdkafka-Configuration/#topic-configuration-properties
216
215
  def with(**args)
217
216
  ensure_active!
218
217
 
@@ -508,7 +507,7 @@ module WaterDrop
508
507
  )
509
508
 
510
509
  # Attempt to reload the producer
511
- idempotent_reload_client_on_fatal_error(@idempotent_fatal_error_attempts)
510
+ idempotent_reload_client_on_fatal_error(@idempotent_fatal_error_attempts, e)
512
511
 
513
512
  # Wait before retrying to avoid rapid reload loops
514
513
  sleep(@config.wait_backoff_on_idempotent_fatal_error / 1_000.0)
@@ -565,5 +564,15 @@ module WaterDrop
565
564
  ensure
566
565
  @operations_in_progress.decrement
567
566
  end
567
+
568
+ # Reloads the client
569
+ # @note This should be used only within proper mutexes internally
570
+ def reload!
571
+ @client.flush(current_variant.max_wait_timeout)
572
+ purge
573
+ @client.close
574
+ @client = nil
575
+ @status.configured!
576
+ end
568
577
  end
569
578
  end
@@ -3,5 +3,5 @@
3
3
  # WaterDrop library
4
4
  module WaterDrop
5
5
  # Current WaterDrop version
6
- VERSION = '2.8.12'
6
+ VERSION = '2.8.14'
7
7
  end
data/lib/waterdrop.rb CHANGED
@@ -42,5 +42,7 @@ loader = Zeitwerk::Loader.for_gem
42
42
  loader.inflector.inflect('waterdrop' => 'WaterDrop')
43
43
  # Do not load vendors instrumentation components. Those need to be required manually if needed
44
44
  loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
45
+ # Do not load testing components. Those need to be required manually in test environments
46
+ loader.ignore("#{__dir__}/waterdrop/producer/testing.rb")
45
47
  loader.setup
46
48
  loader.eager_load
data/waterdrop.gemspec CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
17
17
  spec.licenses = %w[LGPL-3.0-only Commercial]
18
18
 
19
19
  spec.add_dependency 'karafka-core', '>= 2.4.9', '< 3.0.0'
20
- spec.add_dependency 'karafka-rdkafka', '>= 0.20.0'
20
+ spec.add_dependency 'karafka-rdkafka', '>= 0.23.1'
21
21
  spec.add_dependency 'zeitwerk', '~> 2.3'
22
22
 
23
23
  spec.required_ruby_version = '>= 3.2.0'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.8.12
4
+ version: 2.8.14
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -35,14 +35,14 @@ dependencies:
35
35
  requirements:
36
36
  - - ">="
37
37
  - !ruby/object:Gem::Version
38
- version: 0.20.0
38
+ version: 0.23.1
39
39
  type: :runtime
40
40
  prerelease: false
41
41
  version_requirements: !ruby/object:Gem::Requirement
42
42
  requirements:
43
43
  - - ">="
44
44
  - !ruby/object:Gem::Version
45
- version: 0.20.0
45
+ version: 0.23.1
46
46
  - !ruby/object:Gem::Dependency
47
47
  name: zeitwerk
48
48
  requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,6 @@ extensions: []
65
65
  extra_rdoc_files: []
66
66
  files:
67
67
  - ".coditsu/ci.yml"
68
- - ".diffend.yml"
69
68
  - ".github/CODEOWNERS"
70
69
  - ".github/FUNDING.yml"
71
70
  - ".github/ISSUE_TEMPLATE/bug_report.md"
@@ -78,6 +77,7 @@ files:
78
77
  - ".rspec"
79
78
  - ".ruby-gemset"
80
79
  - ".ruby-version"
80
+ - ".yard-lint.yml"
81
81
  - CHANGELOG.md
82
82
  - Gemfile
83
83
  - Gemfile.lock
@@ -123,6 +123,7 @@ files:
123
123
  - lib/waterdrop/producer/idempotence.rb
124
124
  - lib/waterdrop/producer/status.rb
125
125
  - lib/waterdrop/producer/sync.rb
126
+ - lib/waterdrop/producer/testing.rb
126
127
  - lib/waterdrop/producer/transactions.rb
127
128
  - lib/waterdrop/producer/variant.rb
128
129
  - lib/waterdrop/version.rb
data/.diffend.yml DELETED
@@ -1,3 +0,0 @@
1
- project_id: 'ee590bc9-f375-4832-ac8e-beb174b49aa5'
2
- shareable_id: '1325b111-b957-4fd8-bd6f-fead41e06034'
3
- shareable_key: 'd29c6230-8bf6-479b-83ff-98f374d3466e'