breaker_machines 0.9.2-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +184 -0
- data/ext/breaker_machines_native/extconf.rb +3 -0
- data/lib/breaker_machines/async_circuit.rb +47 -0
- data/lib/breaker_machines/async_support.rb +104 -0
- data/lib/breaker_machines/cascading_circuit.rb +177 -0
- data/lib/breaker_machines/circuit/async_state_management.rb +71 -0
- data/lib/breaker_machines/circuit/base.rb +59 -0
- data/lib/breaker_machines/circuit/callbacks.rb +135 -0
- data/lib/breaker_machines/circuit/configuration.rb +67 -0
- data/lib/breaker_machines/circuit/coordinated_state_management.rb +117 -0
- data/lib/breaker_machines/circuit/execution.rb +231 -0
- data/lib/breaker_machines/circuit/hedged_execution.rb +115 -0
- data/lib/breaker_machines/circuit/introspection.rb +93 -0
- data/lib/breaker_machines/circuit/native.rb +127 -0
- data/lib/breaker_machines/circuit/state_callbacks.rb +72 -0
- data/lib/breaker_machines/circuit/state_management.rb +59 -0
- data/lib/breaker_machines/circuit.rb +8 -0
- data/lib/breaker_machines/circuit_group.rb +153 -0
- data/lib/breaker_machines/console.rb +345 -0
- data/lib/breaker_machines/coordinated_circuit.rb +10 -0
- data/lib/breaker_machines/dsl/cascading_circuit_builder.rb +20 -0
- data/lib/breaker_machines/dsl/circuit_builder.rb +209 -0
- data/lib/breaker_machines/dsl/hedged_builder.rb +21 -0
- data/lib/breaker_machines/dsl/parallel_fallback_wrapper.rb +20 -0
- data/lib/breaker_machines/dsl.rb +283 -0
- data/lib/breaker_machines/errors.rb +71 -0
- data/lib/breaker_machines/hedged_async_support.rb +88 -0
- data/lib/breaker_machines/native_extension.rb +81 -0
- data/lib/breaker_machines/native_speedup.rb +10 -0
- data/lib/breaker_machines/registry.rb +243 -0
- data/lib/breaker_machines/storage/backend_state.rb +69 -0
- data/lib/breaker_machines/storage/base.rb +52 -0
- data/lib/breaker_machines/storage/bucket_memory.rb +176 -0
- data/lib/breaker_machines/storage/cache.rb +169 -0
- data/lib/breaker_machines/storage/fallback_chain.rb +294 -0
- data/lib/breaker_machines/storage/memory.rb +140 -0
- data/lib/breaker_machines/storage/native.rb +93 -0
- data/lib/breaker_machines/storage/null.rb +54 -0
- data/lib/breaker_machines/storage.rb +8 -0
- data/lib/breaker_machines/types.rb +41 -0
- data/lib/breaker_machines/version.rb +5 -0
- data/lib/breaker_machines.rb +200 -0
- data/lib/breaker_machines_native/breaker_machines_native.so +0 -0
- data/sig/README.md +74 -0
- data/sig/all.rbs +25 -0
- data/sig/breaker_machines/circuit.rbs +154 -0
- data/sig/breaker_machines/console.rbs +32 -0
- data/sig/breaker_machines/dsl.rbs +50 -0
- data/sig/breaker_machines/errors.rbs +24 -0
- data/sig/breaker_machines/interfaces.rbs +46 -0
- data/sig/breaker_machines/registry.rbs +30 -0
- data/sig/breaker_machines/storage.rbs +65 -0
- data/sig/breaker_machines/types.rbs +97 -0
- data/sig/breaker_machines.rbs +42 -0
- data/sig/manifest.yaml +5 -0
- metadata +224 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'weakref'
|
|
5
|
+
|
|
6
|
+
module BreakerMachines
|
|
7
|
+
# Global registry for tracking all circuit breaker instances
|
|
8
|
+
class Registry
|
|
9
|
+
include Singleton
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
@circuits = Concurrent::Map.new
|
|
13
|
+
@named_circuits = Concurrent::Map.new # For dynamic circuits by name
|
|
14
|
+
@mutex = Mutex.new
|
|
15
|
+
@registration_count = 0
|
|
16
|
+
@cleanup_interval = 100 # Clean up every N registrations
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Register a circuit instance
|
|
20
|
+
def register(circuit)
|
|
21
|
+
@mutex.synchronize do
|
|
22
|
+
# Use circuit object as key - Concurrent::Map handles object identity correctly
|
|
23
|
+
@circuits[circuit] = WeakRef.new(circuit)
|
|
24
|
+
|
|
25
|
+
# Periodic cleanup
|
|
26
|
+
@registration_count += 1
|
|
27
|
+
if @registration_count >= @cleanup_interval
|
|
28
|
+
cleanup_dead_references_unsafe
|
|
29
|
+
@registration_count = 0
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Unregister a circuit instance
|
|
35
|
+
def unregister(circuit)
|
|
36
|
+
@mutex.synchronize do
|
|
37
|
+
@circuits.delete(circuit)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Get all active circuits
|
|
42
|
+
def all_circuits
|
|
43
|
+
@mutex.synchronize do
|
|
44
|
+
@circuits.values.map do |weak_ref|
|
|
45
|
+
weak_ref.__getobj__
|
|
46
|
+
rescue WeakRef::RefError
|
|
47
|
+
nil
|
|
48
|
+
end.compact
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Find circuits by name
|
|
53
|
+
def find_by_name(name)
|
|
54
|
+
all_circuits.select { |circuit| circuit.name == name }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Find first circuit by name
|
|
58
|
+
def find(name)
|
|
59
|
+
find_by_name(name).first
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Force open a circuit by name
|
|
63
|
+
def force_open(name) # rubocop:disable Naming/PredicateMethod
|
|
64
|
+
circuits = find_by_name(name)
|
|
65
|
+
return false if circuits.empty?
|
|
66
|
+
|
|
67
|
+
circuits.each(&:force_open)
|
|
68
|
+
true
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Force close a circuit by name
|
|
72
|
+
def force_close(name) # rubocop:disable Naming/PredicateMethod
|
|
73
|
+
circuits = find_by_name(name)
|
|
74
|
+
return false if circuits.empty?
|
|
75
|
+
|
|
76
|
+
circuits.each(&:force_close)
|
|
77
|
+
true
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Reset a circuit by name
|
|
81
|
+
def reset(name) # rubocop:disable Naming/PredicateMethod
|
|
82
|
+
circuits = find_by_name(name)
|
|
83
|
+
return false if circuits.empty?
|
|
84
|
+
|
|
85
|
+
circuits.each(&:reset)
|
|
86
|
+
true
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Get summary statistics
|
|
90
|
+
def stats_summary
|
|
91
|
+
circuits = all_circuits
|
|
92
|
+
{
|
|
93
|
+
total: circuits.size,
|
|
94
|
+
by_state: circuits.group_by(&:status_name).transform_values(&:count),
|
|
95
|
+
by_name: circuits.group_by(&:name).transform_values(&:count)
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Get all stats with detailed metrics
|
|
100
|
+
def all_stats
|
|
101
|
+
circuits = all_circuits
|
|
102
|
+
|
|
103
|
+
{
|
|
104
|
+
summary: stats_summary,
|
|
105
|
+
circuits: circuits.map { |c| c.stats.to_h },
|
|
106
|
+
health: {
|
|
107
|
+
open_count: circuits.count(&:open?),
|
|
108
|
+
closed_count: circuits.count(&:closed?),
|
|
109
|
+
half_open_count: circuits.count(&:half_open?),
|
|
110
|
+
total_failures: circuits.sum { |c| c.stats.failure_count },
|
|
111
|
+
total_successes: circuits.sum { |c| c.stats.success_count }
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Get detailed information for all circuits
|
|
117
|
+
def detailed_report
|
|
118
|
+
all_circuits.map(&:to_h)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Get or create a globally managed dynamic circuit
|
|
122
|
+
def get_or_create_dynamic_circuit(name, owner, config)
|
|
123
|
+
@mutex.synchronize do
|
|
124
|
+
# Check if circuit already exists and is still alive
|
|
125
|
+
if @named_circuits.key?(name)
|
|
126
|
+
weak_ref = @named_circuits[name]
|
|
127
|
+
begin
|
|
128
|
+
existing_circuit = weak_ref.__getobj__
|
|
129
|
+
return existing_circuit if existing_circuit
|
|
130
|
+
rescue WeakRef::RefError
|
|
131
|
+
# Circuit was garbage collected, remove the stale reference
|
|
132
|
+
@named_circuits.delete(name)
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Create new circuit with weak owner reference
|
|
137
|
+
# Don't auto-register to avoid deadlock
|
|
138
|
+
weak_owner = owner.is_a?(WeakRef) ? owner : WeakRef.new(owner)
|
|
139
|
+
circuit_config = config.merge(owner: weak_owner, auto_register: false)
|
|
140
|
+
new_circuit = Circuit.new(name, circuit_config)
|
|
141
|
+
|
|
142
|
+
# Manually register the circuit (we're already in sync block)
|
|
143
|
+
@circuits[new_circuit] = WeakRef.new(new_circuit)
|
|
144
|
+
@named_circuits[name] = WeakRef.new(new_circuit)
|
|
145
|
+
|
|
146
|
+
new_circuit
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Remove a dynamic circuit by name
|
|
151
|
+
def remove_dynamic_circuit(name)
|
|
152
|
+
@mutex.synchronize do
|
|
153
|
+
if @named_circuits.key?(name)
|
|
154
|
+
weak_ref = @named_circuits.delete(name)
|
|
155
|
+
begin
|
|
156
|
+
circuit = weak_ref.__getobj__
|
|
157
|
+
@circuits.delete(circuit) if circuit
|
|
158
|
+
true
|
|
159
|
+
rescue WeakRef::RefError
|
|
160
|
+
false
|
|
161
|
+
end
|
|
162
|
+
else
|
|
163
|
+
false
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Get all dynamic circuit names
|
|
169
|
+
def dynamic_circuit_names
|
|
170
|
+
@mutex.synchronize do
|
|
171
|
+
alive_names = []
|
|
172
|
+
@named_circuits.each_pair do |name, weak_ref|
|
|
173
|
+
weak_ref.__getobj__
|
|
174
|
+
alive_names << name
|
|
175
|
+
rescue WeakRef::RefError
|
|
176
|
+
@named_circuits.delete(name)
|
|
177
|
+
end
|
|
178
|
+
alive_names
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Cleanup stale dynamic circuits older than given age
|
|
183
|
+
def cleanup_stale_dynamic_circuits(max_age_seconds = 3600)
|
|
184
|
+
@mutex.synchronize do
|
|
185
|
+
cutoff_time = Time.now - max_age_seconds
|
|
186
|
+
stale_names = []
|
|
187
|
+
|
|
188
|
+
@named_circuits.each_pair do |name, weak_ref|
|
|
189
|
+
circuit = weak_ref.__getobj__
|
|
190
|
+
# Check if circuit has a last_activity_time and it's stale
|
|
191
|
+
if circuit.respond_to?(:last_activity_time) &&
|
|
192
|
+
circuit.last_activity_time &&
|
|
193
|
+
circuit.last_activity_time < cutoff_time
|
|
194
|
+
stale_names << name
|
|
195
|
+
end
|
|
196
|
+
rescue WeakRef::RefError
|
|
197
|
+
stale_names << name
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
stale_names.each do |name|
|
|
201
|
+
weak_ref = @named_circuits.delete(name)
|
|
202
|
+
begin
|
|
203
|
+
circuit = weak_ref.__getobj__
|
|
204
|
+
@circuits.delete(circuit) if circuit
|
|
205
|
+
rescue WeakRef::RefError
|
|
206
|
+
# Already gone
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
stale_names.size
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# Clear all circuits (useful for testing)
|
|
215
|
+
def clear
|
|
216
|
+
@mutex.synchronize do
|
|
217
|
+
@circuits.clear
|
|
218
|
+
@named_circuits.clear
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Clean up dead references (thread-safe)
|
|
223
|
+
def cleanup_dead_references
|
|
224
|
+
@mutex.synchronize do
|
|
225
|
+
cleanup_dead_references_unsafe
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
private
|
|
230
|
+
|
|
231
|
+
# Clean up dead references (must be called within mutex)
|
|
232
|
+
def cleanup_dead_references_unsafe
|
|
233
|
+
dead_ids = []
|
|
234
|
+
@circuits.each_pair do |id, weak_ref|
|
|
235
|
+
weak_ref.__getobj__
|
|
236
|
+
rescue WeakRef::RefError
|
|
237
|
+
dead_ids << id
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
dead_ids.each { |id| @circuits.delete(id) }
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BreakerMachines
|
|
4
|
+
module Storage
|
|
5
|
+
# Manages the health state of a single storage backend using a state machine.
|
|
6
|
+
class BackendState
|
|
7
|
+
attr_reader :name, :failure_count, :last_failure_at
|
|
8
|
+
attr_accessor :health
|
|
9
|
+
|
|
10
|
+
def initialize(name, threshold:, timeout:)
|
|
11
|
+
@name = name
|
|
12
|
+
@threshold = threshold
|
|
13
|
+
@timeout = timeout
|
|
14
|
+
@failure_count = 0
|
|
15
|
+
@last_failure_at = nil
|
|
16
|
+
@health = :healthy
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
state_machine :health, initial: :healthy do
|
|
20
|
+
event :trip do
|
|
21
|
+
transition healthy: :unhealthy, if: :threshold_reached?
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
event :recover do
|
|
25
|
+
transition unhealthy: :healthy
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
event :reset do
|
|
29
|
+
transition all => :healthy
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
before_transition to: :unhealthy do |backend, _transition|
|
|
33
|
+
backend.instance_variable_set(:@unhealthy_until,
|
|
34
|
+
BreakerMachines.monotonic_time + backend.instance_variable_get(:@timeout))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
after_transition to: :healthy do |backend, _transition|
|
|
38
|
+
backend.instance_variable_set(:@failure_count, 0)
|
|
39
|
+
backend.instance_variable_set(:@last_failure_at, nil)
|
|
40
|
+
backend.instance_variable_set(:@unhealthy_until, nil)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def record_failure
|
|
45
|
+
@failure_count += 1
|
|
46
|
+
@last_failure_at = BreakerMachines.monotonic_time
|
|
47
|
+
trip
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def threshold_reached?
|
|
51
|
+
@failure_count >= @threshold
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def unhealthy_due_to_timeout?
|
|
55
|
+
return false unless unhealthy?
|
|
56
|
+
|
|
57
|
+
unhealthy_until = instance_variable_get(:@unhealthy_until)
|
|
58
|
+
return false unless unhealthy_until
|
|
59
|
+
|
|
60
|
+
if BreakerMachines.monotonic_time > unhealthy_until
|
|
61
|
+
recover
|
|
62
|
+
false
|
|
63
|
+
else
|
|
64
|
+
true
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BreakerMachines
|
|
4
|
+
module Storage
|
|
5
|
+
# Abstract base class for storage backends
|
|
6
|
+
class Base
|
|
7
|
+
def initialize(**options)
|
|
8
|
+
@options = options
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Status management
|
|
12
|
+
def get_status(circuit_name)
|
|
13
|
+
raise NotImplementedError
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def set_status(circuit_name, status, opened_at = nil)
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Metrics tracking
|
|
21
|
+
def record_success(circuit_name, duration)
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def record_failure(circuit_name, duration)
|
|
26
|
+
raise NotImplementedError
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def success_count(circuit_name, window_seconds)
|
|
30
|
+
raise NotImplementedError
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def failure_count(circuit_name, window_seconds)
|
|
34
|
+
raise NotImplementedError
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Cleanup
|
|
38
|
+
def clear(circuit_name)
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def clear_all
|
|
43
|
+
raise NotImplementedError
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Timeout handling - each backend must implement its own timeout strategy
|
|
47
|
+
def with_timeout(timeout_ms)
|
|
48
|
+
raise NotImplementedError, "#{self.class} must implement #with_timeout to handle #{timeout_ms}ms timeouts"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'concurrent/map'
|
|
4
|
+
require 'concurrent/array'
|
|
5
|
+
|
|
6
|
+
module BreakerMachines
|
|
7
|
+
module Storage
|
|
8
|
+
# Efficient bucket-based memory storage implementation
|
|
9
|
+
# Uses fixed-size circular buffers for constant-time event counting
|
|
10
|
+
#
|
|
11
|
+
# WARNING: This storage backend is NOT compatible with DRb (distributed Ruby)
|
|
12
|
+
# environments as memory is not shared between processes. Use Cache backend
|
|
13
|
+
# with an external cache store (Redis, Memcached) for distributed setups.
|
|
14
|
+
class BucketMemory < Base
|
|
15
|
+
BUCKET_SIZE = 1 # 1 second per bucket
|
|
16
|
+
|
|
17
|
+
def initialize(**options)
|
|
18
|
+
super
|
|
19
|
+
@circuits = Concurrent::Map.new
|
|
20
|
+
@circuit_buckets = Concurrent::Map.new
|
|
21
|
+
@event_logs = Concurrent::Map.new
|
|
22
|
+
@bucket_count = options[:bucket_count] || 300 # Default 5 minutes
|
|
23
|
+
@max_events = options[:max_events] || 100
|
|
24
|
+
# Store creation time as anchor for relative timestamps (like Rust implementation)
|
|
25
|
+
@start_time = BreakerMachines.monotonic_time
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def get_status(circuit_name)
|
|
29
|
+
circuit_data = @circuits[circuit_name]
|
|
30
|
+
return nil unless circuit_data
|
|
31
|
+
|
|
32
|
+
BreakerMachines::Status.new(
|
|
33
|
+
status: circuit_data[:status],
|
|
34
|
+
opened_at: circuit_data[:opened_at]
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def set_status(circuit_name, status, opened_at = nil)
|
|
39
|
+
@circuits[circuit_name] = {
|
|
40
|
+
status: status,
|
|
41
|
+
opened_at: opened_at,
|
|
42
|
+
updated_at: monotonic_time
|
|
43
|
+
}
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def record_success(circuit_name, duration)
|
|
47
|
+
record_event(circuit_name, :success, duration)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def record_failure(circuit_name, duration)
|
|
51
|
+
record_event(circuit_name, :failure, duration)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def success_count(circuit_name, window_seconds)
|
|
55
|
+
count_events(circuit_name, :success, window_seconds)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def failure_count(circuit_name, window_seconds)
|
|
59
|
+
count_events(circuit_name, :failure, window_seconds)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def clear(circuit_name)
|
|
63
|
+
@circuits.delete(circuit_name)
|
|
64
|
+
@circuit_buckets.delete(circuit_name)
|
|
65
|
+
@event_logs.delete(circuit_name)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def clear_all
|
|
69
|
+
@circuits.clear
|
|
70
|
+
@circuit_buckets.clear
|
|
71
|
+
@event_logs.clear
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def record_event_with_details(circuit_name, type, duration, error: nil, new_state: nil)
|
|
75
|
+
events = @event_logs.compute_if_absent(circuit_name) { Concurrent::Array.new }
|
|
76
|
+
|
|
77
|
+
event = {
|
|
78
|
+
type: type,
|
|
79
|
+
timestamp: monotonic_time,
|
|
80
|
+
duration_ms: (duration * 1000).round(2)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
event[:error_class] = error.class.name if error
|
|
84
|
+
event[:error_message] = error.message if error
|
|
85
|
+
event[:new_state] = new_state if new_state
|
|
86
|
+
|
|
87
|
+
events << event
|
|
88
|
+
|
|
89
|
+
# Keep only the most recent events
|
|
90
|
+
events.shift while events.size > @max_events
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def event_log(circuit_name, limit)
|
|
94
|
+
events = @event_logs[circuit_name]
|
|
95
|
+
return [] unless events
|
|
96
|
+
|
|
97
|
+
events.last(limit).map(&:dup)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def record_event(circuit_name, type, _duration)
|
|
103
|
+
buckets = @circuit_buckets.compute_if_absent(circuit_name) do
|
|
104
|
+
{
|
|
105
|
+
successes: Concurrent::Array.new(@bucket_count) { Concurrent::AtomicFixnum.new(0) },
|
|
106
|
+
failures: Concurrent::Array.new(@bucket_count) { Concurrent::AtomicFixnum.new(0) },
|
|
107
|
+
last_bucket_time: Concurrent::AtomicReference.new(current_bucket_time)
|
|
108
|
+
}
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
current_time = current_bucket_time
|
|
112
|
+
rotate_buckets_if_needed(buckets, current_time)
|
|
113
|
+
|
|
114
|
+
bucket_index = current_time % @bucket_count
|
|
115
|
+
counter_array = type == :success ? buckets[:successes] : buckets[:failures]
|
|
116
|
+
counter_array[bucket_index].increment
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def count_events(circuit_name, type, window_seconds)
|
|
120
|
+
buckets = @circuit_buckets[circuit_name]
|
|
121
|
+
return 0 unless buckets
|
|
122
|
+
|
|
123
|
+
current_time = current_bucket_time
|
|
124
|
+
rotate_buckets_if_needed(buckets, current_time)
|
|
125
|
+
|
|
126
|
+
# Calculate how many buckets to count
|
|
127
|
+
buckets_to_count = [window_seconds / BUCKET_SIZE, @bucket_count].min.to_i
|
|
128
|
+
|
|
129
|
+
counter_array = type == :success ? buckets[:successes] : buckets[:failures]
|
|
130
|
+
total = 0
|
|
131
|
+
|
|
132
|
+
buckets_to_count.times do |i|
|
|
133
|
+
bucket_index = (current_time - i) % @bucket_count
|
|
134
|
+
total += counter_array[bucket_index].value
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
total
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def rotate_buckets_if_needed(buckets, current_time)
|
|
141
|
+
last_time = buckets[:last_bucket_time].value
|
|
142
|
+
|
|
143
|
+
return if current_time == last_time
|
|
144
|
+
|
|
145
|
+
# Only one thread should rotate buckets
|
|
146
|
+
return unless buckets[:last_bucket_time].compare_and_set(last_time, current_time)
|
|
147
|
+
|
|
148
|
+
# Clear buckets that are now outdated
|
|
149
|
+
time_diff = current_time - last_time
|
|
150
|
+
buckets_to_clear = [time_diff, @bucket_count].min
|
|
151
|
+
|
|
152
|
+
buckets_to_clear.times do |i|
|
|
153
|
+
# Clear the bucket that will be reused
|
|
154
|
+
bucket_index = (last_time + i + 1) % @bucket_count
|
|
155
|
+
buckets[:successes][bucket_index].value = 0
|
|
156
|
+
buckets[:failures][bucket_index].value = 0
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def current_bucket_time
|
|
161
|
+
(monotonic_time / BUCKET_SIZE).to_i
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def monotonic_time
|
|
165
|
+
# Return time relative to storage creation (matches Rust implementation)
|
|
166
|
+
BreakerMachines.monotonic_time - @start_time
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def with_timeout(_timeout_ms)
|
|
170
|
+
# BucketMemory operations should be instant, but we'll still respect the timeout
|
|
171
|
+
# This is more for consistency and to catch any potential deadlocks
|
|
172
|
+
yield
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BreakerMachines
|
|
4
|
+
module Storage
|
|
5
|
+
# Storage adapter for ActiveSupport::Cache
|
|
6
|
+
# Works with any Rails cache store (Redis, Memcached, Memory, etc.)
|
|
7
|
+
class Cache < Base
|
|
8
|
+
def initialize(cache_store: Rails.cache, **options)
|
|
9
|
+
super(**options)
|
|
10
|
+
@cache = cache_store
|
|
11
|
+
@prefix = options[:prefix] || 'breaker_machines'
|
|
12
|
+
@expires_in = options[:expires_in] || 300 # 5 minutes default
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def get_status(circuit_name)
|
|
16
|
+
data = @cache.read(status_key(circuit_name))
|
|
17
|
+
return nil unless data
|
|
18
|
+
|
|
19
|
+
BreakerMachines::Status.new(
|
|
20
|
+
status: data[:status].to_sym,
|
|
21
|
+
opened_at: data[:opened_at]
|
|
22
|
+
)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def set_status(circuit_name, status, opened_at = nil)
|
|
26
|
+
@cache.write(
|
|
27
|
+
status_key(circuit_name),
|
|
28
|
+
{
|
|
29
|
+
status: status,
|
|
30
|
+
opened_at: opened_at,
|
|
31
|
+
updated_at: monotonic_time
|
|
32
|
+
},
|
|
33
|
+
expires_in: @expires_in
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def record_success(circuit_name, _duration)
|
|
38
|
+
increment_counter(success_key(circuit_name))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def record_failure(circuit_name, _duration)
|
|
42
|
+
increment_counter(failure_key(circuit_name))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def success_count(circuit_name, window_seconds)
|
|
46
|
+
get_window_count(success_key(circuit_name), window_seconds)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def failure_count(circuit_name, window_seconds)
|
|
50
|
+
get_window_count(failure_key(circuit_name), window_seconds)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def clear(circuit_name)
|
|
54
|
+
@cache.delete(status_key(circuit_name))
|
|
55
|
+
@cache.delete(success_key(circuit_name))
|
|
56
|
+
@cache.delete(failure_key(circuit_name))
|
|
57
|
+
@cache.delete(events_key(circuit_name))
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def clear_all
|
|
61
|
+
# Clear all circuit data by pattern if cache supports it
|
|
62
|
+
if @cache.respond_to?(:delete_matched)
|
|
63
|
+
@cache.delete_matched("#{@prefix}:*")
|
|
64
|
+
else
|
|
65
|
+
# Fallback: can't efficiently clear all without pattern support
|
|
66
|
+
BreakerMachines.logger&.warn(
|
|
67
|
+
"[BreakerMachines] Cache store doesn't support delete_matched. " \
|
|
68
|
+
'Individual circuit data must be cleared manually.'
|
|
69
|
+
)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def record_event_with_details(circuit_name, type, duration, error: nil, new_state: nil)
|
|
74
|
+
events = @cache.fetch(events_key(circuit_name)) { [] }
|
|
75
|
+
|
|
76
|
+
event = {
|
|
77
|
+
type: type,
|
|
78
|
+
timestamp: monotonic_time,
|
|
79
|
+
duration_ms: (duration * 1000).round(2)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
event[:error_class] = error.class.name if error
|
|
83
|
+
event[:error_message] = error.message if error
|
|
84
|
+
event[:new_state] = new_state if new_state
|
|
85
|
+
|
|
86
|
+
events << event
|
|
87
|
+
events.shift while events.size > (@max_events || 100)
|
|
88
|
+
|
|
89
|
+
@cache.write(events_key(circuit_name), events, expires_in: @expires_in)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def event_log(circuit_name, limit)
|
|
93
|
+
events = @cache.read(events_key(circuit_name)) || []
|
|
94
|
+
events.last(limit)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def with_timeout(_timeout_ms)
|
|
98
|
+
# Rails cache operations should rely on their own underlying timeouts
|
|
99
|
+
# Using Ruby's Timeout.timeout is dangerous and can cause deadlocks
|
|
100
|
+
# For Redis cache stores, configure connect_timeout and read_timeout instead
|
|
101
|
+
yield
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
def increment_counter(key)
|
|
107
|
+
# Use increment if available, otherwise fetch-and-update
|
|
108
|
+
if @cache.respond_to?(:increment)
|
|
109
|
+
@cache.increment(key, 1, expires_in: @expires_in)
|
|
110
|
+
else
|
|
111
|
+
# Fallback for caches without atomic increment
|
|
112
|
+
current = @cache.fetch(key) { {} }
|
|
113
|
+
current[current_bucket] = (current[current_bucket] || 0) + 1
|
|
114
|
+
prune_old_buckets(current)
|
|
115
|
+
@cache.write(key, current, expires_in: @expires_in)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def get_window_count(key, window_seconds)
|
|
120
|
+
if @cache.respond_to?(:increment)
|
|
121
|
+
# For simple counter-based caches, we can't get windowed counts
|
|
122
|
+
# Would need to implement bucketing similar to fallback
|
|
123
|
+
@cache.read(key) || 0
|
|
124
|
+
else
|
|
125
|
+
# Bucket-based counting for accurate windows
|
|
126
|
+
buckets = @cache.read(key) || {}
|
|
127
|
+
current_time = current_bucket
|
|
128
|
+
|
|
129
|
+
total = 0
|
|
130
|
+
window_seconds.times do |i|
|
|
131
|
+
bucket_key = current_time - i
|
|
132
|
+
total += buckets[bucket_key] || 0
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
total
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def prune_old_buckets(buckets)
|
|
140
|
+
cutoff = current_bucket - 300 # Keep 5 minutes of data
|
|
141
|
+
buckets.delete_if { |time, _| time < cutoff }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def current_bucket
|
|
145
|
+
Time.now.to_i
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def status_key(circuit_name)
|
|
149
|
+
"#{@prefix}:#{circuit_name}:status"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def success_key(circuit_name)
|
|
153
|
+
"#{@prefix}:#{circuit_name}:successes"
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def failure_key(circuit_name)
|
|
157
|
+
"#{@prefix}:#{circuit_name}:failures"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def events_key(circuit_name)
|
|
161
|
+
"#{@prefix}:#{circuit_name}:events"
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def monotonic_time
|
|
165
|
+
BreakerMachines.monotonic_time
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|