breaker_machines 0.22.1-aarch64-linux → 0.24.0-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/breaker_machines/async_support.rb +1 -0
- data/lib/breaker_machines/circuit/admission.rb +23 -18
- data/lib/breaker_machines/circuit/base.rb +1 -0
- data/lib/breaker_machines/circuit/callbacks.rb +27 -0
- data/lib/breaker_machines/circuit/execution.rb +10 -5
- data/lib/breaker_machines/circuit/hedged_execution.rb +11 -4
- data/lib/breaker_machines/circuit/introspection.rb +1 -1
- data/lib/breaker_machines/circuit/native.rb +100 -25
- data/lib/breaker_machines/circuit/state_callbacks.rb +12 -17
- data/lib/breaker_machines/hedged_async_support.rb +16 -13
- data/lib/breaker_machines/registry.rb +37 -57
- data/lib/breaker_machines/storage/native.rb +3 -12
- data/lib/breaker_machines/version.rb +1 -1
- data/lib/breaker_machines.rb +2 -0
- data/lib/breaker_machines_native/breaker_machines_native.so +0 -0
- metadata +20 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 306115fb41ffaaebc459975b4e3901615f4ff665895db186393c988094ab5f41
|
|
4
|
+
data.tar.gz: 42b6478b1649b161c2fc16294cc75a35becbcaae6b335aa31a2932aa230ecdb8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a273bf860b0c1a086a46070dd8d21d8c0e82348d96c95347cdf88ae691dfa223fa27e8bb1e1aad62d5d23a1761841fa9a282f92de159f7d4a0cc29a898686d2a
|
|
7
|
+
data.tar.gz: 3353d5ccb58ec51f856edc4f6e6be04632b9b5e52c9cd183ad636559bccbbf64e6972ab4b87cc2af88eb8bea5e8ac363b45511b71f798892dca14adecb4805c9
|
|
@@ -31,6 +31,7 @@ module BreakerMachines
|
|
|
31
31
|
rescue StandardError => e
|
|
32
32
|
# Re-raise if it's not an async timeout or configured exception
|
|
33
33
|
raise unless e.is_a?(async_timeout_error_class) || @config[:exceptions].any? { |klass| e.is_a?(klass) }
|
|
34
|
+
raise if caller_gave_up?(e)
|
|
34
35
|
|
|
35
36
|
complete_call_failure(admission, start_time, e)
|
|
36
37
|
completed = true
|
|
@@ -53,9 +53,10 @@ module BreakerMachines
|
|
|
53
53
|
|
|
54
54
|
def admit_half_open_call
|
|
55
55
|
state_epoch = @state_epoch.value
|
|
56
|
-
|
|
56
|
+
# Slots taken, less those already banked as successes.
|
|
57
|
+
probes_in_flight = @half_open_attempts.increment - @half_open_successes.value
|
|
57
58
|
|
|
58
|
-
return Call.new(state: :half_open, state_epoch:) if
|
|
59
|
+
return Call.new(state: :half_open, state_epoch:) if probes_in_flight <= @config[:half_open_calls]
|
|
59
60
|
|
|
60
61
|
# This caller lost the race for a probe slot.
|
|
61
62
|
@half_open_attempts.decrement
|
|
@@ -69,28 +70,32 @@ module BreakerMachines
|
|
|
69
70
|
def handle_success(admission)
|
|
70
71
|
return unless admission.half_open?
|
|
71
72
|
|
|
72
|
-
|
|
73
|
-
|
|
73
|
+
with_deferred_callbacks do
|
|
74
|
+
with_refreshed_status do
|
|
75
|
+
return unless admission_current?(admission) && half_open?
|
|
74
76
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
+
successful_attempts = @half_open_successes.increment
|
|
78
|
+
next unless successful_attempts >= @config[:success_threshold]
|
|
77
79
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
@half_open_attempts.value = 0
|
|
81
|
+
@half_open_successes.value = 0
|
|
82
|
+
reset
|
|
83
|
+
end
|
|
81
84
|
end
|
|
82
85
|
end
|
|
83
86
|
|
|
84
87
|
def handle_failure(admission)
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
88
|
+
with_deferred_callbacks do
|
|
89
|
+
with_refreshed_status do
|
|
90
|
+
return unless admission_current?(admission)
|
|
91
|
+
|
|
92
|
+
if admission.closed? && closed? && failure_threshold_exceeded?
|
|
93
|
+
trip
|
|
94
|
+
elsif admission.half_open? && half_open?
|
|
95
|
+
@half_open_attempts.value = 0
|
|
96
|
+
@half_open_successes.value = 0
|
|
97
|
+
trip
|
|
98
|
+
end
|
|
94
99
|
end
|
|
95
100
|
end
|
|
96
101
|
end
|
|
@@ -38,6 +38,7 @@ module BreakerMachines
|
|
|
38
38
|
|
|
39
39
|
def initialize_runtime_state
|
|
40
40
|
@opened_at = Concurrent::AtomicReference.new(nil)
|
|
41
|
+
@open_timeout = Concurrent::AtomicReference.new(nil)
|
|
41
42
|
@half_open_attempts = Concurrent::AtomicFixnum.new(0)
|
|
42
43
|
@half_open_successes = Concurrent::AtomicFixnum.new(0)
|
|
43
44
|
@state_epoch = Concurrent::AtomicFixnum.new(0)
|
|
@@ -9,7 +9,32 @@ module BreakerMachines
|
|
|
9
9
|
|
|
10
10
|
private
|
|
11
11
|
|
|
12
|
+
DEFERRED = :breaker_machines_deferred_callbacks
|
|
13
|
+
|
|
14
|
+
# A state-change callback is user code. Running it under the circuit's
|
|
15
|
+
# write lock lets it block every other caller, and letting it raise
|
|
16
|
+
# replaces the operation's own error, so it is collected here and run
|
|
17
|
+
# after the lock is released.
|
|
12
18
|
def invoke_callback(callback_name)
|
|
19
|
+
pending = Thread.current[DEFERRED]
|
|
20
|
+
return pending << callback_name if pending
|
|
21
|
+
|
|
22
|
+
run_callback(callback_name)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def with_deferred_callbacks
|
|
26
|
+
outer = Thread.current[DEFERRED]
|
|
27
|
+
pending = []
|
|
28
|
+
Thread.current[DEFERRED] = pending
|
|
29
|
+
begin
|
|
30
|
+
yield
|
|
31
|
+
ensure
|
|
32
|
+
Thread.current[DEFERRED] = outer
|
|
33
|
+
pending.each { |name| run_callback(name) }
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def run_callback(callback_name)
|
|
13
38
|
callback = @config[callback_name]
|
|
14
39
|
return unless callback
|
|
15
40
|
|
|
@@ -26,6 +51,8 @@ module BreakerMachines
|
|
|
26
51
|
else
|
|
27
52
|
callback.call
|
|
28
53
|
end
|
|
54
|
+
rescue StandardError => e
|
|
55
|
+
BreakerMachines.instrument('callback_error', circuit: @name, callback: callback_name, error: e)
|
|
29
56
|
end
|
|
30
57
|
|
|
31
58
|
def invoke_fallback(error)
|
|
@@ -32,6 +32,7 @@ module BreakerMachines
|
|
|
32
32
|
private
|
|
33
33
|
|
|
34
34
|
def execute_with_state_check(&)
|
|
35
|
+
DeadlineMachines.check!
|
|
35
36
|
attempt_recovery_if_ready
|
|
36
37
|
|
|
37
38
|
# Apply bulkheading first, outside of any locks
|
|
@@ -78,6 +79,8 @@ module BreakerMachines
|
|
|
78
79
|
completed = true
|
|
79
80
|
result
|
|
80
81
|
rescue *@config[:exceptions] => e
|
|
82
|
+
raise if caller_gave_up?(e)
|
|
83
|
+
|
|
81
84
|
complete_call_failure(admission, start_time, e)
|
|
82
85
|
completed = true
|
|
83
86
|
raise unless @config[:fallback]
|
|
@@ -88,6 +91,13 @@ module BreakerMachines
|
|
|
88
91
|
end
|
|
89
92
|
end
|
|
90
93
|
|
|
94
|
+
# A caller abandoning its own budget says nothing about the dependency's
|
|
95
|
+
# health, so it must not count toward the failure threshold. An inner
|
|
96
|
+
# scope expiring is a genuine timeout and still does.
|
|
97
|
+
def caller_gave_up?(error)
|
|
98
|
+
error.is_a?(DeadlineMachines::Error) && DeadlineMachines.current&.done?
|
|
99
|
+
end
|
|
100
|
+
|
|
91
101
|
def execute_sync_operation(&)
|
|
92
102
|
warn_about_sync_timeout
|
|
93
103
|
return execute_hedged(&) if @config[:hedged_requests] || @config[:backends]
|
|
@@ -157,11 +167,6 @@ module BreakerMachines
|
|
|
157
167
|
end
|
|
158
168
|
end
|
|
159
169
|
|
|
160
|
-
def success_threshold_reached?
|
|
161
|
-
recent_successes = @storage.success_count(@name, @config[:failure_window])
|
|
162
|
-
recent_successes >= @config[:success_threshold]
|
|
163
|
-
end
|
|
164
|
-
|
|
165
170
|
def record_success(duration)
|
|
166
171
|
@metrics&.record_success(@name, duration)
|
|
167
172
|
@storage&.record_success(@name, duration)
|
|
@@ -12,6 +12,12 @@ module BreakerMachines
|
|
|
12
12
|
extend ActiveSupport::Concern
|
|
13
13
|
|
|
14
14
|
# Execute a hedged request pattern
|
|
15
|
+
# Seconds a hedge waits when the circuit configures no timeout.
|
|
16
|
+
HEDGE_TIMEOUT = 30
|
|
17
|
+
|
|
18
|
+
# Seconds between polls of the result queue.
|
|
19
|
+
HEDGE_POLL = 0.001
|
|
20
|
+
|
|
15
21
|
def execute_hedged(&)
|
|
16
22
|
return execute_single_hedged(&) unless @config[:backends]&.any?
|
|
17
23
|
|
|
@@ -55,7 +61,7 @@ module BreakerMachines
|
|
|
55
61
|
|
|
56
62
|
callables.each_with_index do |callable, index|
|
|
57
63
|
# Add delay for hedge requests (not the first one)
|
|
58
|
-
sleep(delay_ms / 1000.0) if index.positive? && delay_ms.positive?
|
|
64
|
+
sleep(DeadlineMachines.clamp_sleep(delay_ms / 1000.0)) if index.positive? && delay_ms.positive?
|
|
59
65
|
|
|
60
66
|
# Skip if already got a result
|
|
61
67
|
break if cancelled.value
|
|
@@ -75,7 +81,8 @@ module BreakerMachines
|
|
|
75
81
|
|
|
76
82
|
# Wait for first result or all errors
|
|
77
83
|
begin
|
|
78
|
-
|
|
84
|
+
DeadlineMachines.check!
|
|
85
|
+
Timeout.timeout(DeadlineMachines.clamp_sleep(@config[:timeout] || HEDGE_TIMEOUT)) do
|
|
79
86
|
# Check for successful result
|
|
80
87
|
loop do
|
|
81
88
|
unless result_queue.empty?
|
|
@@ -87,8 +94,8 @@ module BreakerMachines
|
|
|
87
94
|
# Check if all requests failed
|
|
88
95
|
raise error_queue.pop if error_queue.size >= callables.size
|
|
89
96
|
|
|
90
|
-
|
|
91
|
-
sleep
|
|
97
|
+
DeadlineMachines.check!
|
|
98
|
+
sleep HEDGE_POLL
|
|
92
99
|
end
|
|
93
100
|
end
|
|
94
101
|
ensure
|
|
@@ -67,7 +67,7 @@ module BreakerMachines
|
|
|
67
67
|
"It opened at #{opened_time} after #{@config[:failure_threshold]} failures.#{error_info}"
|
|
68
68
|
when :half_open
|
|
69
69
|
"Circuit '#{@name}' is HALF-OPEN. Testing with limited requests " \
|
|
70
|
-
"(#{@
|
|
70
|
+
"(#{@half_open_successes.value}/#{@config[:success_threshold]} successes)."
|
|
71
71
|
end
|
|
72
72
|
|
|
73
73
|
# Add cascade information if this is a cascading circuit
|
|
@@ -5,8 +5,9 @@ module BreakerMachines
|
|
|
5
5
|
# Native circuit breaker implementation using Rust FFI
|
|
6
6
|
#
|
|
7
7
|
# This provides a high-performance circuit breaker with state machine logic
|
|
8
|
-
# implemented in Rust. It
|
|
9
|
-
#
|
|
8
|
+
# implemented in Rust. It answers the same questions as the pure-Ruby
|
|
9
|
+
# circuit under the same names, so the global registry reports on both
|
|
10
|
+
# kinds of circuit alike.
|
|
10
11
|
#
|
|
11
12
|
# @example Basic usage
|
|
12
13
|
# circuit = BreakerMachines::Circuit::Native.new('api_calls',
|
|
@@ -31,27 +32,32 @@ module BreakerMachines
|
|
|
31
32
|
# @option options [Integer] :failure_threshold Number of failures to open circuit (default: 5)
|
|
32
33
|
# @option options [Float] :failure_window_secs Time window for counting failures (default: 60.0)
|
|
33
34
|
# @option options [Float] :half_open_timeout_secs Timeout before attempting reset (default: 30.0)
|
|
35
|
+
# @option options [Float] :jitter_factor Cooldown jitter, 0.0-1.0 (default: 0.0)
|
|
34
36
|
# @option options [Integer] :success_threshold Successes needed to close from half-open (default: 2)
|
|
37
|
+
# @option options [Integer] :half_open_calls Probes admitted at a time while half-open (default: 1)
|
|
35
38
|
# @option options [Boolean] :auto_register Register with global registry (default: true)
|
|
39
|
+
# @raise [ConfigurationError] for an option the native circuit cannot honour
|
|
36
40
|
def initialize(name, options = {})
|
|
37
41
|
unless BreakerMachines.native_available?
|
|
38
42
|
raise BreakerMachines::ConfigurationError,
|
|
39
43
|
'Native extension not available. Install with native support or use Circuit::Ruby'
|
|
40
44
|
end
|
|
41
45
|
|
|
46
|
+
unhonoured = options.keys - default_config.keys
|
|
47
|
+
if unhonoured.any?
|
|
48
|
+
raise BreakerMachines::ConfigurationError,
|
|
49
|
+
"Circuit::Native cannot honour #{unhonoured.join(', ')}; use BreakerMachines::Circuit for those"
|
|
50
|
+
end
|
|
51
|
+
|
|
42
52
|
@name = name
|
|
43
53
|
@config = default_config.merge(options)
|
|
44
54
|
|
|
45
55
|
# Create the native circuit breaker
|
|
46
|
-
@native_circuit = BreakerMachinesNative::Circuit.new(
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
half_open_timeout_secs: @config[:half_open_timeout_secs],
|
|
52
|
-
success_threshold: @config[:success_threshold]
|
|
53
|
-
}
|
|
54
|
-
)
|
|
56
|
+
@native_circuit = BreakerMachinesNative::Circuit.new(name, native_config)
|
|
57
|
+
|
|
58
|
+
# Zero of the native circuit's own clock, which its timestamps count from.
|
|
59
|
+
@clock_origin = BreakerMachines.monotonic_time
|
|
60
|
+
@epoch_origin = BreakerMachines.monotonic_to_epoch(@clock_origin)
|
|
55
61
|
|
|
56
62
|
# Register with global registry unless disabled
|
|
57
63
|
BreakerMachines::Registry.instance.register(self) unless @config[:auto_register] == false
|
|
@@ -88,40 +94,109 @@ module BreakerMachines
|
|
|
88
94
|
@native_circuit.is_half_open
|
|
89
95
|
end
|
|
90
96
|
|
|
91
|
-
#
|
|
92
|
-
# @return [
|
|
93
|
-
def
|
|
94
|
-
@native_circuit.state_name
|
|
97
|
+
# Current state name
|
|
98
|
+
# @return [Symbol] :closed, :open, or :half_open
|
|
99
|
+
def status_name
|
|
100
|
+
@native_circuit.state_name.to_sym
|
|
95
101
|
end
|
|
96
102
|
|
|
97
|
-
#
|
|
98
|
-
def
|
|
99
|
-
|
|
103
|
+
# @return [BreakerMachines::Stats] counts over the configured failure window
|
|
104
|
+
def stats
|
|
105
|
+
BreakerMachines::Stats.new(
|
|
106
|
+
state: status_name,
|
|
107
|
+
failure_count: @native_circuit.failure_count,
|
|
108
|
+
success_count: @native_circuit.success_count,
|
|
109
|
+
last_failure_at: nil,
|
|
110
|
+
opened_at: opened_at,
|
|
111
|
+
half_open_attempts: @native_circuit.half_open_attempts,
|
|
112
|
+
half_open_successes: @native_circuit.half_open_successes
|
|
113
|
+
)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# @return [Hash] a copy of this circuit's configuration
|
|
117
|
+
def configuration
|
|
118
|
+
@config.dup
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# @param limit [Integer] how many of the most recent outcomes to return
|
|
122
|
+
# @return [Array<Hash>] recorded outcomes, oldest first
|
|
123
|
+
def event_log(limit: 20)
|
|
124
|
+
@native_circuit.event_log(limit).map do |event|
|
|
125
|
+
event.merge(timestamp_epoch: @epoch_origin + event[:timestamp])
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# The native circuit records outcomes, not the errors behind them.
|
|
130
|
+
# @return [nil]
|
|
131
|
+
def last_error
|
|
132
|
+
nil
|
|
100
133
|
end
|
|
101
134
|
|
|
102
|
-
#
|
|
103
|
-
|
|
104
|
-
|
|
135
|
+
# @return [nil] (see #last_error)
|
|
136
|
+
def last_error_info
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# @return [Hash] full state, statistics and configuration
|
|
141
|
+
def to_h
|
|
105
142
|
{
|
|
106
143
|
name: @name,
|
|
107
|
-
state:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
144
|
+
state: status_name,
|
|
145
|
+
stats: stats.to_h,
|
|
146
|
+
config: configuration,
|
|
147
|
+
event_log: event_log,
|
|
148
|
+
last_error: last_error_info&.to_h
|
|
111
149
|
}
|
|
112
150
|
end
|
|
113
151
|
|
|
152
|
+
# Open the circuit whatever its failure count
|
|
153
|
+
# @return [true]
|
|
154
|
+
def force_open
|
|
155
|
+
@native_circuit.force_open
|
|
156
|
+
true
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Close the circuit, keeping the events in its failure window
|
|
160
|
+
# @return [true]
|
|
161
|
+
def force_close
|
|
162
|
+
@native_circuit.force_close
|
|
163
|
+
true
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
alias reset force_close
|
|
167
|
+
alias reset! force_close
|
|
168
|
+
|
|
169
|
+
# Close the circuit and clear its failure window
|
|
170
|
+
# @return [true]
|
|
171
|
+
def hard_reset
|
|
172
|
+
@native_circuit.reset
|
|
173
|
+
true
|
|
174
|
+
end
|
|
175
|
+
|
|
114
176
|
private
|
|
115
177
|
|
|
178
|
+
# Native timestamps count from the circuit's own clock; Stats reports
|
|
179
|
+
# BreakerMachines.monotonic_time.
|
|
180
|
+
def opened_at
|
|
181
|
+
opened = @native_circuit.opened_at
|
|
182
|
+
opened && @clock_origin + opened
|
|
183
|
+
end
|
|
184
|
+
|
|
116
185
|
def default_config
|
|
117
186
|
{
|
|
118
187
|
failure_threshold: 5,
|
|
119
188
|
failure_window_secs: 60.0,
|
|
120
189
|
half_open_timeout_secs: 30.0,
|
|
190
|
+
jitter_factor: 0.0,
|
|
121
191
|
success_threshold: 2,
|
|
192
|
+
half_open_calls: 1,
|
|
122
193
|
auto_register: true
|
|
123
194
|
}
|
|
124
195
|
end
|
|
196
|
+
|
|
197
|
+
def native_config
|
|
198
|
+
@config.except(:auto_register)
|
|
199
|
+
end
|
|
125
200
|
end
|
|
126
201
|
end
|
|
127
202
|
end
|
|
@@ -11,6 +11,7 @@ module BreakerMachines
|
|
|
11
11
|
def on_circuit_open
|
|
12
12
|
@state_epoch.increment
|
|
13
13
|
@opened_at.value = BreakerMachines.monotonic_time
|
|
14
|
+
@open_timeout.value = draw_reset_timeout
|
|
14
15
|
persist_status(:open, @opened_at.value)
|
|
15
16
|
if @storage.respond_to?(:record_event_with_details)
|
|
16
17
|
@storage.record_event_with_details(@name, :state_change, 0,
|
|
@@ -23,6 +24,7 @@ module BreakerMachines
|
|
|
23
24
|
def on_circuit_close
|
|
24
25
|
@state_epoch.increment
|
|
25
26
|
@opened_at.value = nil
|
|
27
|
+
@open_timeout.value = nil
|
|
26
28
|
@last_error.value = nil
|
|
27
29
|
@last_failure_at.value = nil
|
|
28
30
|
persist_status(:closed)
|
|
@@ -50,26 +52,19 @@ module BreakerMachines
|
|
|
50
52
|
def reset_timeout_elapsed?
|
|
51
53
|
return false unless @opened_at.value
|
|
52
54
|
|
|
53
|
-
|
|
54
|
-
# This matches the Rust implementation which uses chrono-machines for jitter
|
|
55
|
-
timeout_with_jitter = if (jitter_factor = @config[:reset_timeout_jitter]) && jitter_factor.positive?
|
|
56
|
-
calculate_timeout_with_jitter(@config[:reset_timeout], jitter_factor)
|
|
57
|
-
else
|
|
58
|
-
@config[:reset_timeout]
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
BreakerMachines.monotonic_time - @opened_at.value >= timeout_with_jitter
|
|
55
|
+
BreakerMachines.monotonic_time - @opened_at.value >= (@open_timeout.value || draw_reset_timeout)
|
|
62
56
|
end
|
|
63
57
|
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
58
|
+
# Drawn once per opening. Re-drawing on every check would return the
|
|
59
|
+
# minimum of N samples, pinning the cooldown to the bottom of its range
|
|
60
|
+
# and synchronising a fleet instead of spreading it.
|
|
61
|
+
def draw_reset_timeout
|
|
62
|
+
jitter_factor = @config[:reset_timeout_jitter]
|
|
63
|
+
return @config[:reset_timeout] unless jitter_factor&.positive?
|
|
64
|
+
|
|
65
|
+
ChronoMachines.jitter(@config[:reset_timeout], factor: jitter_factor)
|
|
72
66
|
end
|
|
67
|
+
|
|
73
68
|
end
|
|
74
69
|
end
|
|
75
70
|
end
|
|
@@ -47,40 +47,43 @@ module BreakerMachines
|
|
|
47
47
|
# Race callables; return first result or raise if it was an Exception
|
|
48
48
|
# Uses modern Async::Promise and Async::Barrier for cleaner synchronization
|
|
49
49
|
# @param callables [Array<Proc>] Tasks to race
|
|
50
|
+
# One hedge's outcome, so a returned exception object is not mistaken
|
|
51
|
+
# for a failure.
|
|
52
|
+
Hedge = Data.define(:value, :error)
|
|
53
|
+
|
|
50
54
|
# @param delay_ms [Integer] Delay in milliseconds between task starts
|
|
51
55
|
# @return [Object] First successful result
|
|
52
56
|
# @raise [Exception] The first exception received
|
|
53
57
|
def race_tasks(callables, delay_ms: 0)
|
|
54
58
|
promise = Async::Promise.new
|
|
55
59
|
barrier = Async::Barrier.new
|
|
60
|
+
outstanding = callables.size
|
|
61
|
+
failure = nil
|
|
56
62
|
|
|
57
63
|
begin
|
|
58
|
-
|
|
64
|
+
outcome = Async do
|
|
59
65
|
callables.each_with_index do |callable, idx|
|
|
60
66
|
barrier.async do
|
|
61
|
-
|
|
62
|
-
sleep(delay_ms / 1000.0) if idx.positive? && delay_ms.positive?
|
|
67
|
+
sleep(DeadlineMachines.clamp_sleep(delay_ms / 1000.0)) if idx.positive? && delay_ms.positive?
|
|
63
68
|
|
|
64
69
|
begin
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
# Only the first resolution will succeed
|
|
68
|
-
promise.resolve(result) unless promise.resolved?
|
|
70
|
+
value = callable.call
|
|
71
|
+
promise.resolve(Hedge.new(value, nil)) unless promise.resolved?
|
|
69
72
|
rescue StandardError => e
|
|
70
|
-
|
|
71
|
-
|
|
73
|
+
failure = e
|
|
74
|
+
outstanding -= 1
|
|
75
|
+
promise.resolve(Hedge.new(nil, e)) if outstanding.zero? && !promise.resolved?
|
|
72
76
|
end
|
|
73
77
|
end
|
|
74
78
|
end
|
|
75
79
|
|
|
76
|
-
# Wait for the first resolution (either success or exception)
|
|
77
80
|
promise.wait
|
|
78
81
|
end.wait
|
|
79
82
|
|
|
80
|
-
|
|
81
|
-
|
|
83
|
+
raise outcome.error if outcome.error
|
|
84
|
+
|
|
85
|
+
outcome.value
|
|
82
86
|
ensure
|
|
83
|
-
# Ensure all tasks are stopped
|
|
84
87
|
barrier&.stop
|
|
85
88
|
end
|
|
86
89
|
end
|
|
@@ -11,6 +11,7 @@ module BreakerMachines
|
|
|
11
11
|
def initialize
|
|
12
12
|
@circuits = Concurrent::Map.new
|
|
13
13
|
@named_circuits = Concurrent::Map.new # For dynamic circuits by name
|
|
14
|
+
@named_last_used = Concurrent::Map.new # Monotonic time each was last handed out
|
|
14
15
|
@mutex = Mutex.new
|
|
15
16
|
@registration_count = 0
|
|
16
17
|
@cleanup_interval = 100 # Clean up every N registrations
|
|
@@ -19,13 +20,13 @@ module BreakerMachines
|
|
|
19
20
|
# Register a circuit instance
|
|
20
21
|
def register(circuit)
|
|
21
22
|
@mutex.synchronize do
|
|
22
|
-
|
|
23
|
-
@circuits[circuit] = WeakRef.new(circuit)
|
|
23
|
+
@circuits[circuit.object_id] = WeakRef.new(circuit)
|
|
24
24
|
|
|
25
25
|
# Periodic cleanup
|
|
26
26
|
@registration_count += 1
|
|
27
27
|
if @registration_count >= @cleanup_interval
|
|
28
28
|
cleanup_dead_references_unsafe
|
|
29
|
+
cleanup_stale_dynamic_circuits_unsafe(DYNAMIC_CIRCUIT_MAX_AGE)
|
|
29
30
|
@registration_count = 0
|
|
30
31
|
end
|
|
31
32
|
end
|
|
@@ -34,7 +35,7 @@ module BreakerMachines
|
|
|
34
35
|
# Unregister a circuit instance
|
|
35
36
|
def unregister(circuit)
|
|
36
37
|
@mutex.synchronize do
|
|
37
|
-
@circuits.delete(circuit)
|
|
38
|
+
@circuits.delete(circuit.object_id)
|
|
38
39
|
end
|
|
39
40
|
end
|
|
40
41
|
|
|
@@ -122,15 +123,10 @@ module BreakerMachines
|
|
|
122
123
|
def get_or_create_dynamic_circuit(name, owner, config)
|
|
123
124
|
@mutex.synchronize do
|
|
124
125
|
# Check if circuit already exists and is still alive
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
return existing_circuit if existing_circuit
|
|
130
|
-
rescue WeakRef::RefError
|
|
131
|
-
# Circuit was garbage collected, remove the stale reference
|
|
132
|
-
@named_circuits.delete(name)
|
|
133
|
-
end
|
|
126
|
+
existing = @named_circuits[name]
|
|
127
|
+
if existing
|
|
128
|
+
@named_last_used[name] = BreakerMachines.monotonic_time
|
|
129
|
+
return existing
|
|
134
130
|
end
|
|
135
131
|
|
|
136
132
|
# Create new circuit with weak owner reference
|
|
@@ -139,9 +135,9 @@ module BreakerMachines
|
|
|
139
135
|
circuit_config = config.merge(owner: weak_owner, auto_register: false)
|
|
140
136
|
new_circuit = Circuit.new(name, circuit_config)
|
|
141
137
|
|
|
142
|
-
|
|
143
|
-
@
|
|
144
|
-
@
|
|
138
|
+
@circuits[new_circuit.object_id] = WeakRef.new(new_circuit)
|
|
139
|
+
@named_circuits[name] = new_circuit
|
|
140
|
+
@named_last_used[name] = BreakerMachines.monotonic_time
|
|
145
141
|
|
|
146
142
|
new_circuit
|
|
147
143
|
end
|
|
@@ -151,14 +147,10 @@ module BreakerMachines
|
|
|
151
147
|
def remove_dynamic_circuit(name)
|
|
152
148
|
@mutex.synchronize do
|
|
153
149
|
if @named_circuits.key?(name)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
true
|
|
159
|
-
rescue WeakRef::RefError
|
|
160
|
-
false
|
|
161
|
-
end
|
|
150
|
+
circuit = @named_circuits.delete(name)
|
|
151
|
+
@named_last_used.delete(name)
|
|
152
|
+
@circuits.delete(circuit.object_id) if circuit
|
|
153
|
+
true
|
|
162
154
|
else
|
|
163
155
|
false
|
|
164
156
|
end
|
|
@@ -168,47 +160,34 @@ module BreakerMachines
|
|
|
168
160
|
# Get all dynamic circuit names
|
|
169
161
|
def dynamic_circuit_names
|
|
170
162
|
@mutex.synchronize do
|
|
171
|
-
|
|
172
|
-
@named_circuits.each_pair do |name, weak_ref|
|
|
173
|
-
weak_ref.__getobj__
|
|
174
|
-
alive_names << name
|
|
175
|
-
rescue WeakRef::RefError
|
|
176
|
-
@named_circuits.delete(name)
|
|
177
|
-
end
|
|
178
|
-
alive_names
|
|
163
|
+
@named_circuits.keys
|
|
179
164
|
end
|
|
180
165
|
end
|
|
181
166
|
|
|
182
167
|
# Cleanup stale dynamic circuits older than given age
|
|
183
|
-
def cleanup_stale_dynamic_circuits(max_age_seconds =
|
|
184
|
-
@mutex.synchronize
|
|
185
|
-
|
|
186
|
-
stale_names = []
|
|
187
|
-
|
|
188
|
-
@named_circuits.each_pair do |name, weak_ref|
|
|
189
|
-
circuit = weak_ref.__getobj__
|
|
190
|
-
# Check if circuit has a last_activity_time and it's stale
|
|
191
|
-
if circuit.respond_to?(:last_activity_time) &&
|
|
192
|
-
circuit.last_activity_time &&
|
|
193
|
-
circuit.last_activity_time < cutoff_time
|
|
194
|
-
stale_names << name
|
|
195
|
-
end
|
|
196
|
-
rescue WeakRef::RefError
|
|
197
|
-
stale_names << name
|
|
198
|
-
end
|
|
168
|
+
def cleanup_stale_dynamic_circuits(max_age_seconds = DYNAMIC_CIRCUIT_MAX_AGE)
|
|
169
|
+
@mutex.synchronize { cleanup_stale_dynamic_circuits_unsafe(max_age_seconds) }
|
|
170
|
+
end
|
|
199
171
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
# Already gone
|
|
207
|
-
end
|
|
208
|
-
end
|
|
172
|
+
# Seconds a named dynamic circuit is kept after its last activity.
|
|
173
|
+
DYNAMIC_CIRCUIT_MAX_AGE = 3600
|
|
174
|
+
|
|
175
|
+
private def cleanup_stale_dynamic_circuits_unsafe(max_age_seconds)
|
|
176
|
+
cutoff_time = BreakerMachines.monotonic_time - max_age_seconds
|
|
177
|
+
stale_names = []
|
|
209
178
|
|
|
210
|
-
|
|
179
|
+
@named_circuits.each_pair do |name, _circuit|
|
|
180
|
+
last_used = @named_last_used[name]
|
|
181
|
+
stale_names << name if last_used.nil? || last_used < cutoff_time
|
|
211
182
|
end
|
|
183
|
+
|
|
184
|
+
stale_names.each do |name|
|
|
185
|
+
circuit = @named_circuits.delete(name)
|
|
186
|
+
@named_last_used.delete(name)
|
|
187
|
+
@circuits.delete(circuit.object_id) if circuit
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
stale_names.size
|
|
212
191
|
end
|
|
213
192
|
|
|
214
193
|
# Clear all circuits (useful for testing)
|
|
@@ -216,6 +195,7 @@ module BreakerMachines
|
|
|
216
195
|
@mutex.synchronize do
|
|
217
196
|
@circuits.clear
|
|
218
197
|
@named_circuits.clear
|
|
198
|
+
@named_last_used.clear
|
|
219
199
|
end
|
|
220
200
|
end
|
|
221
201
|
|
|
@@ -68,18 +68,9 @@ module BreakerMachines
|
|
|
68
68
|
@backend.clear_all
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
when :success
|
|
75
|
-
record_success(circuit_name, duration)
|
|
76
|
-
when :failure
|
|
77
|
-
record_failure(circuit_name, duration)
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
# NOTE: Error and state details not tracked in native backend
|
|
81
|
-
# This is intentional for performance - use Memory backend if you need full event details
|
|
82
|
-
end
|
|
71
|
+
# The caller has already recorded the outcome; this backend keeps no
|
|
72
|
+
# error or state detail, so there is nothing further to store.
|
|
73
|
+
def record_event_with_details(_circuit_name, _type, _duration, error: nil, new_state: nil); end
|
|
83
74
|
|
|
84
75
|
def event_log(circuit_name, limit)
|
|
85
76
|
@backend.event_log(circuit_name.to_s, limit).map do |event|
|
data/lib/breaker_machines.rb
CHANGED
|
Binary file
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: breaker_machines
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.24.0
|
|
5
5
|
platform: aarch64-linux
|
|
6
6
|
authors:
|
|
7
7
|
- Abdelkader Boudih
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -28,16 +28,30 @@ dependencies:
|
|
|
28
28
|
name: chrono_machines
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
|
-
- - "
|
|
31
|
+
- - ">="
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version:
|
|
33
|
+
version: 0.12.1
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
|
-
- - "
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: 0.12.1
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: deadline_machines
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0.1'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
39
53
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '0.
|
|
54
|
+
version: '0.1'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
56
|
name: concurrent-ruby
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|