breaker_machines 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -3
- data/ext/breaker_machines_native/Cargo.toml +8 -0
- data/ext/breaker_machines_native/core/Cargo.toml +18 -0
- data/ext/breaker_machines_native/core/examples/basic.rs +61 -0
- data/ext/breaker_machines_native/core/src/builder.rs +232 -0
- data/ext/breaker_machines_native/core/src/bulkhead.rs +223 -0
- data/ext/breaker_machines_native/core/src/callbacks.rs +58 -0
- data/ext/breaker_machines_native/core/src/circuit.rs +1156 -0
- data/ext/breaker_machines_native/core/src/classifier.rs +177 -0
- data/ext/breaker_machines_native/core/src/errors.rs +47 -0
- data/ext/breaker_machines_native/core/src/lib.rs +62 -0
- data/ext/breaker_machines_native/core/src/storage.rs +377 -0
- data/ext/breaker_machines_native/extconf.rb +40 -0
- data/ext/breaker_machines_native/ffi/Cargo.toml +16 -0
- data/ext/breaker_machines_native/ffi/src/lib.rs +218 -0
- data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/common.rs +355 -0
- data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/dynamic.rs +276 -0
- data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/macros.rs +49 -0
- data/ext/breaker_machines_native/target/debug/build/rb-sys-2bb7281aac8faec8/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
- data/ext/breaker_machines_native/target/debug/build/rb-sys-54cb99ea6aeab8bc/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
- data/ext/breaker_machines_native/target/debug/build/rb-sys-9e64a270c6421e93/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
- data/ext/breaker_machines_native/target/debug/build/rb-sys-e627030114d3fc19/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/Cargo.toml +48 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/examples/basic.rs +61 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/builder.rs +154 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/callbacks.rs +55 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/circuit.rs +607 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/errors.rs +38 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/lib.rs +58 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/storage.rs +377 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/Cargo.toml +48 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/examples/basic.rs +61 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/builder.rs +173 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/callbacks.rs +55 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/circuit.rs +855 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/errors.rs +38 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/lib.rs +58 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/storage.rs +377 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/Cargo.toml +48 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/examples/basic.rs +61 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/builder.rs +154 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/callbacks.rs +55 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/circuit.rs +607 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/errors.rs +38 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/lib.rs +58 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/storage.rs +377 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/Cargo.toml +48 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/examples/basic.rs +61 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/builder.rs +232 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/bulkhead.rs +223 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/callbacks.rs +58 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/circuit.rs +1156 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/classifier.rs +177 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/errors.rs +47 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/lib.rs +62 -0
- data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/storage.rs +377 -0
- data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/common.rs +355 -0
- data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/dynamic.rs +276 -0
- data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/macros.rs +49 -0
- data/ext/breaker_machines_native/target/release/build/rb-sys-064bf9961dd17810/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
- data/lib/breaker_machines/async_circuit.rb +47 -0
- data/lib/breaker_machines/async_support.rb +4 -3
- data/lib/breaker_machines/cascading_circuit.rb +5 -3
- data/lib/breaker_machines/circuit/async_state_management.rb +71 -0
- data/lib/breaker_machines/circuit/base.rb +59 -0
- data/lib/breaker_machines/circuit/callbacks.rb +7 -12
- data/lib/breaker_machines/circuit/configuration.rb +6 -26
- data/lib/breaker_machines/circuit/coordinated_state_management.rb +117 -0
- data/lib/breaker_machines/circuit/hedged_execution.rb +115 -0
- data/lib/breaker_machines/circuit/introspection.rb +1 -0
- data/lib/breaker_machines/circuit/native.rb +127 -0
- data/lib/breaker_machines/circuit/state_callbacks.rb +72 -0
- data/lib/breaker_machines/circuit/state_management.rb +14 -61
- data/lib/breaker_machines/circuit.rb +1 -7
- data/lib/breaker_machines/circuit_group.rb +153 -0
- data/lib/breaker_machines/coordinated_circuit.rb +10 -0
- data/lib/breaker_machines/dsl.rb +2 -2
- data/lib/breaker_machines/errors.rb +20 -0
- data/lib/breaker_machines/hedged_async_support.rb +29 -36
- data/lib/breaker_machines/native_extension.rb +36 -0
- data/lib/breaker_machines/native_speedup.rb +6 -0
- data/lib/breaker_machines/storage/bucket_memory.rb +4 -1
- data/lib/breaker_machines/storage/memory.rb +4 -1
- data/lib/breaker_machines/storage/native.rb +90 -0
- data/lib/breaker_machines/version.rb +1 -1
- data/lib/breaker_machines.rb +115 -11
- data/lib/breaker_machines_native/breaker_machines_native.bundle +0 -0
- data/sig/breaker_machines.rbs +20 -8
- metadata +107 -7
- data/lib/breaker_machines/hedged_execution.rb +0 -113
|
@@ -0,0 +1,855 @@
|
|
|
1
|
+
//! Circuit breaker implementation using state machines
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides a complete circuit breaker with state management.
|
|
4
|
+
|
|
5
|
+
use crate::{callbacks::Callbacks, errors::CircuitError, StorageBackend};
|
|
6
|
+
use state_machines::state_machine;
|
|
7
|
+
use std::sync::Arc;
|
|
8
|
+
|
|
9
|
+
/// Circuit breaker configuration
|
|
10
|
+
#[derive(Debug, Clone)]
|
|
11
|
+
pub struct Config {
|
|
12
|
+
/// Number of failures required to open the circuit (absolute count)
|
|
13
|
+
/// If None, only rate-based threshold is used
|
|
14
|
+
pub failure_threshold: Option<usize>,
|
|
15
|
+
|
|
16
|
+
/// Failure rate threshold (0.0-1.0) - percentage of failures to open circuit
|
|
17
|
+
/// If None, only absolute count threshold is used
|
|
18
|
+
pub failure_rate_threshold: Option<f64>,
|
|
19
|
+
|
|
20
|
+
/// Minimum number of calls before rate-based threshold is evaluated
|
|
21
|
+
pub minimum_calls: usize,
|
|
22
|
+
|
|
23
|
+
/// Time window in seconds for counting failures
|
|
24
|
+
pub failure_window_secs: f64,
|
|
25
|
+
|
|
26
|
+
/// Timeout in seconds before transitioning from Open to HalfOpen
|
|
27
|
+
pub half_open_timeout_secs: f64,
|
|
28
|
+
|
|
29
|
+
/// Number of successes required in HalfOpen to close the circuit
|
|
30
|
+
pub success_threshold: usize,
|
|
31
|
+
|
|
32
|
+
/// Jitter factor for half_open_timeout (0.0 = no jitter, 1.0 = full jitter)
|
|
33
|
+
/// Uses chrono-machines formula: timeout * (1 - jitter + rand * jitter)
|
|
34
|
+
pub jitter_factor: f64,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
impl Default for Config {
|
|
38
|
+
fn default() -> Self {
|
|
39
|
+
Self {
|
|
40
|
+
failure_threshold: Some(5),
|
|
41
|
+
failure_rate_threshold: None,
|
|
42
|
+
minimum_calls: 20,
|
|
43
|
+
failure_window_secs: 60.0,
|
|
44
|
+
half_open_timeout_secs: 30.0,
|
|
45
|
+
success_threshold: 2,
|
|
46
|
+
jitter_factor: 0.0,
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// Context provided to fallback closures when circuit is open
|
|
52
|
+
#[derive(Debug, Clone)]
|
|
53
|
+
pub struct FallbackContext {
|
|
54
|
+
/// Circuit name
|
|
55
|
+
pub circuit_name: String,
|
|
56
|
+
/// Timestamp when circuit opened
|
|
57
|
+
pub opened_at: f64,
|
|
58
|
+
/// Current circuit state
|
|
59
|
+
pub state: &'static str,
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/// Options for circuit breaker calls
|
|
63
|
+
pub struct CallOptions<T, E> {
|
|
64
|
+
/// Optional fallback function called when circuit is open
|
|
65
|
+
pub fallback: Option<Box<dyn FnOnce(&FallbackContext) -> Result<T, E> + Send>>,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
impl<T, E> Default for CallOptions<T, E> {
|
|
69
|
+
fn default() -> Self {
|
|
70
|
+
Self { fallback: None }
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
impl<T, E> CallOptions<T, E> {
|
|
75
|
+
/// Create new call options with no fallback
|
|
76
|
+
pub fn new() -> Self {
|
|
77
|
+
Self::default()
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/// Set a fallback function
|
|
81
|
+
pub fn with_fallback<F>(mut self, f: F) -> Self
|
|
82
|
+
where
|
|
83
|
+
F: FnOnce(&FallbackContext) -> Result<T, E> + Send + 'static,
|
|
84
|
+
{
|
|
85
|
+
self.fallback = Some(Box::new(f));
|
|
86
|
+
self
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/// Trait for converting into CallOptions - allows flexible call() API
|
|
91
|
+
pub trait IntoCallOptions<T, E> {
|
|
92
|
+
fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Implement for plain closures (backward compatibility)
|
|
96
|
+
impl<T, E, F> IntoCallOptions<T, E> for F
|
|
97
|
+
where
|
|
98
|
+
F: FnOnce() -> Result<T, E> + 'static,
|
|
99
|
+
{
|
|
100
|
+
fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>) {
|
|
101
|
+
(Box::new(self), CallOptions::default())
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Implement for (closure, CallOptions) tuple
|
|
106
|
+
impl<T, E, F> IntoCallOptions<T, E> for (F, CallOptions<T, E>)
|
|
107
|
+
where
|
|
108
|
+
F: FnOnce() -> Result<T, E> + 'static,
|
|
109
|
+
{
|
|
110
|
+
fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>) {
|
|
111
|
+
(Box::new(self.0), self.1)
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/// Circuit breaker context - shared data across all states
|
|
116
|
+
#[derive(Clone)]
|
|
117
|
+
pub struct CircuitContext {
|
|
118
|
+
pub name: String,
|
|
119
|
+
pub config: Config,
|
|
120
|
+
pub storage: Arc<dyn StorageBackend>,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
impl Default for CircuitContext {
|
|
124
|
+
fn default() -> Self {
|
|
125
|
+
Self {
|
|
126
|
+
name: String::new(),
|
|
127
|
+
config: Config::default(),
|
|
128
|
+
storage: Arc::new(crate::MemoryStorage::new()),
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
impl std::fmt::Debug for CircuitContext {
|
|
134
|
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
135
|
+
f.debug_struct("CircuitContext")
|
|
136
|
+
.field("name", &self.name)
|
|
137
|
+
.field("config", &self.config)
|
|
138
|
+
.field("storage", &"<dyn StorageBackend>")
|
|
139
|
+
.finish()
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/// Data specific to the Open state
|
|
144
|
+
#[derive(Debug, Clone, Default)]
|
|
145
|
+
pub struct OpenData {
|
|
146
|
+
pub opened_at: f64,
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// Data specific to the HalfOpen state
|
|
150
|
+
#[derive(Debug, Clone, Default)]
|
|
151
|
+
pub struct HalfOpenData {
|
|
152
|
+
pub consecutive_successes: usize,
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Define the circuit breaker state machine with dynamic mode
|
|
156
|
+
state_machine! {
|
|
157
|
+
name: Circuit,
|
|
158
|
+
context: CircuitContext,
|
|
159
|
+
dynamic: true, // Enable dynamic mode for runtime state transitions
|
|
160
|
+
|
|
161
|
+
initial: Closed,
|
|
162
|
+
states: [
|
|
163
|
+
Closed,
|
|
164
|
+
Open(OpenData),
|
|
165
|
+
HalfOpen(HalfOpenData),
|
|
166
|
+
],
|
|
167
|
+
events {
|
|
168
|
+
trip {
|
|
169
|
+
guards: [should_open],
|
|
170
|
+
transition: { from: [Closed, HalfOpen], to: Open }
|
|
171
|
+
}
|
|
172
|
+
attempt_reset {
|
|
173
|
+
guards: [timeout_elapsed],
|
|
174
|
+
transition: { from: Open, to: HalfOpen }
|
|
175
|
+
}
|
|
176
|
+
close {
|
|
177
|
+
guards: [should_close],
|
|
178
|
+
transition: { from: HalfOpen, to: Closed }
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Guards for dynamic mode - implemented on typestate machines
|
|
184
|
+
impl Circuit<Closed> {
|
|
185
|
+
/// Check if failure threshold is exceeded (absolute count or rate-based)
|
|
186
|
+
fn should_open(&self, ctx: &CircuitContext) -> bool {
|
|
187
|
+
let failures = ctx
|
|
188
|
+
.storage
|
|
189
|
+
.failure_count(&ctx.name, ctx.config.failure_window_secs);
|
|
190
|
+
|
|
191
|
+
// Check absolute count threshold
|
|
192
|
+
if let Some(threshold) = ctx.config.failure_threshold {
|
|
193
|
+
if failures >= threshold {
|
|
194
|
+
return true;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Check rate-based threshold
|
|
199
|
+
if let Some(rate_threshold) = ctx.config.failure_rate_threshold {
|
|
200
|
+
let successes = ctx
|
|
201
|
+
.storage
|
|
202
|
+
.success_count(&ctx.name, ctx.config.failure_window_secs);
|
|
203
|
+
let total = failures + successes;
|
|
204
|
+
|
|
205
|
+
// Only evaluate rate if we have minimum calls
|
|
206
|
+
if total >= ctx.config.minimum_calls {
|
|
207
|
+
let failure_rate = if total > 0 {
|
|
208
|
+
failures as f64 / total as f64
|
|
209
|
+
} else {
|
|
210
|
+
0.0
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
if failure_rate >= rate_threshold {
|
|
214
|
+
return true;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
false
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
impl Circuit<HalfOpen> {
|
|
224
|
+
/// Check if failure threshold is exceeded (absolute count or rate-based)
|
|
225
|
+
fn should_open(&self, ctx: &CircuitContext) -> bool {
|
|
226
|
+
let failures = ctx
|
|
227
|
+
.storage
|
|
228
|
+
.failure_count(&ctx.name, ctx.config.failure_window_secs);
|
|
229
|
+
|
|
230
|
+
// Check absolute count threshold
|
|
231
|
+
if let Some(threshold) = ctx.config.failure_threshold {
|
|
232
|
+
if failures >= threshold {
|
|
233
|
+
return true;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Check rate-based threshold
|
|
238
|
+
if let Some(rate_threshold) = ctx.config.failure_rate_threshold {
|
|
239
|
+
let successes = ctx
|
|
240
|
+
.storage
|
|
241
|
+
.success_count(&ctx.name, ctx.config.failure_window_secs);
|
|
242
|
+
let total = failures + successes;
|
|
243
|
+
|
|
244
|
+
// Only evaluate rate if we have minimum calls
|
|
245
|
+
if total >= ctx.config.minimum_calls {
|
|
246
|
+
let failure_rate = if total > 0 {
|
|
247
|
+
failures as f64 / total as f64
|
|
248
|
+
} else {
|
|
249
|
+
0.0
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
if failure_rate >= rate_threshold {
|
|
253
|
+
return true;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
false
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/// Check if enough successes to close circuit
|
|
262
|
+
fn should_close(&self, ctx: &CircuitContext) -> bool {
|
|
263
|
+
let data = self
|
|
264
|
+
.state_data_half_open()
|
|
265
|
+
.expect("HalfOpen state must have data");
|
|
266
|
+
data.consecutive_successes >= ctx.config.success_threshold
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
impl Circuit<Open> {
|
|
271
|
+
/// Check if timeout has elapsed for Open -> HalfOpen transition
|
|
272
|
+
fn timeout_elapsed(&self, ctx: &CircuitContext) -> bool {
|
|
273
|
+
let data = self.state_data_open().expect("Open state must have data");
|
|
274
|
+
let current_time = ctx.storage.monotonic_time();
|
|
275
|
+
let elapsed = current_time - data.opened_at;
|
|
276
|
+
|
|
277
|
+
// Apply jitter using chrono-machines if jitter_factor > 0
|
|
278
|
+
let timeout_secs = if ctx.config.jitter_factor > 0.0 {
|
|
279
|
+
let policy = chrono_machines::Policy {
|
|
280
|
+
max_attempts: 1,
|
|
281
|
+
base_delay_ms: (ctx.config.half_open_timeout_secs * 1000.0) as u64,
|
|
282
|
+
multiplier: 1.0,
|
|
283
|
+
max_delay_ms: (ctx.config.half_open_timeout_secs * 1000.0) as u64,
|
|
284
|
+
};
|
|
285
|
+
let timeout_ms = policy.calculate_delay(1, ctx.config.jitter_factor);
|
|
286
|
+
(timeout_ms as f64) / 1000.0
|
|
287
|
+
} else {
|
|
288
|
+
ctx.config.half_open_timeout_secs
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
elapsed >= timeout_secs
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/// Circuit breaker public API
|
|
296
|
+
pub struct CircuitBreaker {
|
|
297
|
+
machine: DynamicCircuit,
|
|
298
|
+
context: CircuitContext,
|
|
299
|
+
callbacks: Callbacks,
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
impl CircuitBreaker {
|
|
303
|
+
/// Create a new circuit breaker (use builder() for more options)
|
|
304
|
+
pub fn new(name: String, config: Config) -> Self {
|
|
305
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
306
|
+
let context = CircuitContext {
|
|
307
|
+
name,
|
|
308
|
+
config,
|
|
309
|
+
storage,
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
let machine = DynamicCircuit::new(context.clone());
|
|
313
|
+
let callbacks = Callbacks::new();
|
|
314
|
+
|
|
315
|
+
Self {
|
|
316
|
+
machine,
|
|
317
|
+
context,
|
|
318
|
+
callbacks,
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/// Create a circuit breaker with custom context and callbacks (used by builder)
|
|
323
|
+
pub(crate) fn with_context_and_callbacks(
|
|
324
|
+
context: CircuitContext,
|
|
325
|
+
callbacks: Callbacks,
|
|
326
|
+
) -> Self {
|
|
327
|
+
let machine = DynamicCircuit::new(context.clone());
|
|
328
|
+
|
|
329
|
+
Self {
|
|
330
|
+
machine,
|
|
331
|
+
context,
|
|
332
|
+
callbacks,
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/// Create a new circuit breaker builder
|
|
337
|
+
pub fn builder(name: impl Into<String>) -> crate::builder::CircuitBuilder {
|
|
338
|
+
crate::builder::CircuitBuilder::new(name)
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/// Execute a fallible operation with circuit breaker protection
|
|
342
|
+
///
|
|
343
|
+
/// Accepts either:
|
|
344
|
+
/// - A plain closure: `circuit.call(|| api_request())`
|
|
345
|
+
/// - A closure with options: `circuit.call((|| api_request(), CallOptions::new().with_fallback(...)))`
|
|
346
|
+
pub fn call<I, T, E>(&mut self, input: I) -> Result<T, CircuitError<E>>
|
|
347
|
+
where
|
|
348
|
+
I: IntoCallOptions<T, E>,
|
|
349
|
+
{
|
|
350
|
+
let (f, options) = input.into_call_options();
|
|
351
|
+
|
|
352
|
+
// Check for timeout-based Open -> HalfOpen transition
|
|
353
|
+
if self.machine.current_state() == "Open" {
|
|
354
|
+
let _ = self.machine.handle(CircuitEvent::AttemptReset);
|
|
355
|
+
if self.machine.current_state() == "HalfOpen" {
|
|
356
|
+
self.callbacks.trigger_half_open(&self.context.name);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Handle based on current state
|
|
361
|
+
match self.machine.current_state() {
|
|
362
|
+
"Open" => {
|
|
363
|
+
let opened_at = self.machine.open_data().map(|d| d.opened_at).unwrap_or(0.0);
|
|
364
|
+
|
|
365
|
+
// If fallback is provided, use it instead of returning error
|
|
366
|
+
if let Some(fallback) = options.fallback {
|
|
367
|
+
let ctx = FallbackContext {
|
|
368
|
+
circuit_name: self.context.name.clone(),
|
|
369
|
+
opened_at,
|
|
370
|
+
state: "Open",
|
|
371
|
+
};
|
|
372
|
+
return fallback(&ctx).map_err(CircuitError::Execution);
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
Err(CircuitError::Open {
|
|
376
|
+
circuit: self.context.name.clone(),
|
|
377
|
+
opened_at,
|
|
378
|
+
})
|
|
379
|
+
}
|
|
380
|
+
"HalfOpen" => {
|
|
381
|
+
// Check if we've reached the success threshold
|
|
382
|
+
if let Some(data) = self.machine.half_open_data() {
|
|
383
|
+
if data.consecutive_successes >= self.context.config.success_threshold {
|
|
384
|
+
return Err(CircuitError::HalfOpenLimitReached {
|
|
385
|
+
circuit: self.context.name.clone(),
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
self.execute_call(f)
|
|
390
|
+
}
|
|
391
|
+
_ => self.execute_call(f),
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
fn execute_call<T, E>(
|
|
396
|
+
&mut self,
|
|
397
|
+
f: Box<dyn FnOnce() -> Result<T, E>>,
|
|
398
|
+
) -> Result<T, CircuitError<E>> {
|
|
399
|
+
let start = self.context.storage.monotonic_time();
|
|
400
|
+
|
|
401
|
+
match f() {
|
|
402
|
+
Ok(val) => {
|
|
403
|
+
let duration = self.context.storage.monotonic_time() - start;
|
|
404
|
+
self.context
|
|
405
|
+
.storage
|
|
406
|
+
.record_success(&self.context.name, duration);
|
|
407
|
+
|
|
408
|
+
// Handle success in HalfOpen state
|
|
409
|
+
if self.machine.current_state() == "HalfOpen" {
|
|
410
|
+
if let Some(data) = self.machine.half_open_data_mut() {
|
|
411
|
+
data.consecutive_successes += 1;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Try to close the circuit
|
|
415
|
+
if self.machine.handle(CircuitEvent::Close).is_ok() {
|
|
416
|
+
self.callbacks.trigger_close(&self.context.name);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
Ok(val)
|
|
421
|
+
}
|
|
422
|
+
Err(e) => {
|
|
423
|
+
let duration = self.context.storage.monotonic_time() - start;
|
|
424
|
+
self.context
|
|
425
|
+
.storage
|
|
426
|
+
.record_failure(&self.context.name, duration);
|
|
427
|
+
|
|
428
|
+
// Try to trip the circuit
|
|
429
|
+
let result = self.machine.handle(CircuitEvent::Trip);
|
|
430
|
+
if result.is_ok() {
|
|
431
|
+
// Transition succeeded - update opened_at timestamp
|
|
432
|
+
if let Some(data) = self.machine.open_data_mut() {
|
|
433
|
+
data.opened_at = self.context.storage.monotonic_time();
|
|
434
|
+
}
|
|
435
|
+
self.callbacks.trigger_open(&self.context.name);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
Err(CircuitError::Execution(e))
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/// Record a successful operation (for manual tracking)
|
|
444
|
+
pub fn record_success(&self, duration: f64) {
|
|
445
|
+
self.context
|
|
446
|
+
.storage
|
|
447
|
+
.record_success(&self.context.name, duration);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/// Record a failed operation (for manual tracking)
|
|
451
|
+
pub fn record_failure(&self, duration: f64) {
|
|
452
|
+
self.context
|
|
453
|
+
.storage
|
|
454
|
+
.record_failure(&self.context.name, duration);
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/// Check if circuit is open
|
|
458
|
+
pub fn is_open(&self) -> bool {
|
|
459
|
+
self.machine.current_state() == "Open"
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/// Check if circuit is closed
|
|
463
|
+
pub fn is_closed(&self) -> bool {
|
|
464
|
+
self.machine.current_state() == "Closed"
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/// Get current state name
|
|
468
|
+
pub fn state_name(&self) -> &'static str {
|
|
469
|
+
self.machine.current_state()
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/// Clear all events and reset circuit to Closed state
|
|
473
|
+
pub fn reset(&mut self) {
|
|
474
|
+
self.context.storage.clear(&self.context.name);
|
|
475
|
+
// Recreate machine in Closed state
|
|
476
|
+
self.machine = DynamicCircuit::new(self.context.clone());
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
#[cfg(test)]
|
|
481
|
+
mod tests {
|
|
482
|
+
use super::*;
|
|
483
|
+
|
|
484
|
+
#[test]
|
|
485
|
+
fn test_circuit_breaker_creation() {
|
|
486
|
+
let config = Config::default();
|
|
487
|
+
let circuit = CircuitBreaker::new("test".to_string(), config);
|
|
488
|
+
|
|
489
|
+
assert!(circuit.is_closed());
|
|
490
|
+
assert!(!circuit.is_open());
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
#[test]
|
|
494
|
+
fn test_circuit_opens_after_threshold() {
|
|
495
|
+
let config = Config {
|
|
496
|
+
failure_threshold: Some(3),
|
|
497
|
+
..Default::default()
|
|
498
|
+
};
|
|
499
|
+
|
|
500
|
+
let mut circuit = CircuitBreaker::new("test".to_string(), config);
|
|
501
|
+
|
|
502
|
+
// Trigger failures via call() method
|
|
503
|
+
let _ = circuit.call(|| Err::<(), _>("error 1"));
|
|
504
|
+
let _ = circuit.call(|| Err::<(), _>("error 2"));
|
|
505
|
+
assert!(circuit.is_closed());
|
|
506
|
+
|
|
507
|
+
let _ = circuit.call(|| Err::<(), _>("error 3"));
|
|
508
|
+
assert!(circuit.is_open());
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
#[test]
|
|
512
|
+
fn test_reset_clears_state() {
|
|
513
|
+
let config = Config {
|
|
514
|
+
failure_threshold: Some(2),
|
|
515
|
+
..Default::default()
|
|
516
|
+
};
|
|
517
|
+
|
|
518
|
+
let mut circuit = CircuitBreaker::new("test".to_string(), config);
|
|
519
|
+
|
|
520
|
+
// Trigger failures
|
|
521
|
+
let _ = circuit.call(|| Err::<(), _>("error 1"));
|
|
522
|
+
let _ = circuit.call(|| Err::<(), _>("error 2"));
|
|
523
|
+
assert!(circuit.is_open());
|
|
524
|
+
|
|
525
|
+
circuit.reset();
|
|
526
|
+
assert!(circuit.is_closed());
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
#[test]
|
|
530
|
+
fn test_state_machine_closed_to_open_transition() {
|
|
531
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
532
|
+
let config = Config {
|
|
533
|
+
failure_threshold: Some(3),
|
|
534
|
+
..Default::default()
|
|
535
|
+
};
|
|
536
|
+
|
|
537
|
+
let ctx = CircuitContext {
|
|
538
|
+
name: "test_circuit".to_string(),
|
|
539
|
+
config,
|
|
540
|
+
storage: storage.clone(),
|
|
541
|
+
};
|
|
542
|
+
|
|
543
|
+
let mut circuit = DynamicCircuit::new(ctx.clone());
|
|
544
|
+
|
|
545
|
+
// Initially closed - trip should fail guard
|
|
546
|
+
let result = circuit.handle(CircuitEvent::Trip);
|
|
547
|
+
assert!(result.is_err(), "Should fail guard when below threshold");
|
|
548
|
+
|
|
549
|
+
// Record failures to exceed threshold
|
|
550
|
+
storage.record_failure("test_circuit", 0.1);
|
|
551
|
+
storage.record_failure("test_circuit", 0.1);
|
|
552
|
+
storage.record_failure("test_circuit", 0.1);
|
|
553
|
+
|
|
554
|
+
// Now trip should succeed - guards pass
|
|
555
|
+
circuit
|
|
556
|
+
.handle(CircuitEvent::Trip)
|
|
557
|
+
.expect("Should open after reaching threshold");
|
|
558
|
+
|
|
559
|
+
assert_eq!(circuit.current_state(), "Open");
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
#[test]
|
|
563
|
+
fn test_state_machine_open_to_half_open_transition() {
|
|
564
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
565
|
+
let config = Config {
|
|
566
|
+
failure_threshold: Some(2),
|
|
567
|
+
half_open_timeout_secs: 0.001, // Very short timeout for testing
|
|
568
|
+
..Default::default()
|
|
569
|
+
};
|
|
570
|
+
|
|
571
|
+
let ctx = CircuitContext {
|
|
572
|
+
name: "test_circuit".to_string(),
|
|
573
|
+
config,
|
|
574
|
+
storage: storage.clone(),
|
|
575
|
+
};
|
|
576
|
+
|
|
577
|
+
// Record failures and open circuit
|
|
578
|
+
storage.record_failure("test_circuit", 0.1);
|
|
579
|
+
storage.record_failure("test_circuit", 0.1);
|
|
580
|
+
|
|
581
|
+
let mut circuit = DynamicCircuit::new(ctx.clone());
|
|
582
|
+
circuit.handle(CircuitEvent::Trip).expect("Should open");
|
|
583
|
+
|
|
584
|
+
// Set opened_at timestamp
|
|
585
|
+
if let Some(data) = circuit.open_data_mut() {
|
|
586
|
+
data.opened_at = storage.monotonic_time();
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
// Immediately try to reset - should fail guard (timeout not elapsed)
|
|
590
|
+
let result = circuit.handle(CircuitEvent::AttemptReset);
|
|
591
|
+
assert!(
|
|
592
|
+
result.is_err(),
|
|
593
|
+
"Should fail guard when timeout not elapsed"
|
|
594
|
+
);
|
|
595
|
+
|
|
596
|
+
// Wait for timeout
|
|
597
|
+
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
598
|
+
|
|
599
|
+
circuit
|
|
600
|
+
.handle(CircuitEvent::AttemptReset)
|
|
601
|
+
.expect("Should reset after timeout");
|
|
602
|
+
|
|
603
|
+
// Verify we're in HalfOpen state
|
|
604
|
+
assert_eq!(circuit.current_state(), "HalfOpen");
|
|
605
|
+
let data = circuit.half_open_data().expect("Should have HalfOpen data");
|
|
606
|
+
assert_eq!(data.consecutive_successes, 0);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
#[test]
|
|
610
|
+
fn test_state_machine_half_open_to_closed_guard() {
|
|
611
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
612
|
+
let config = Config {
|
|
613
|
+
failure_threshold: Some(2),
|
|
614
|
+
half_open_timeout_secs: 0.001,
|
|
615
|
+
..Default::default()
|
|
616
|
+
};
|
|
617
|
+
|
|
618
|
+
let ctx = CircuitContext {
|
|
619
|
+
name: "test_circuit".to_string(),
|
|
620
|
+
config,
|
|
621
|
+
storage: storage.clone(),
|
|
622
|
+
};
|
|
623
|
+
|
|
624
|
+
// Get to HalfOpen state
|
|
625
|
+
storage.record_failure("test_circuit", 0.1);
|
|
626
|
+
storage.record_failure("test_circuit", 0.1);
|
|
627
|
+
|
|
628
|
+
let mut circuit = DynamicCircuit::new(ctx.clone());
|
|
629
|
+
circuit.handle(CircuitEvent::Trip).expect("Should open");
|
|
630
|
+
|
|
631
|
+
// Set opened_at and wait for timeout
|
|
632
|
+
if let Some(data) = circuit.open_data_mut() {
|
|
633
|
+
data.opened_at = storage.monotonic_time();
|
|
634
|
+
}
|
|
635
|
+
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
636
|
+
|
|
637
|
+
circuit
|
|
638
|
+
.handle(CircuitEvent::AttemptReset)
|
|
639
|
+
.expect("Should reset");
|
|
640
|
+
|
|
641
|
+
// Try to close - should fail guard (not enough successes)
|
|
642
|
+
let result = circuit.handle(CircuitEvent::Close);
|
|
643
|
+
assert!(result.is_err(), "Should fail guard without successes");
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
#[test]
|
|
647
|
+
fn test_jitter_disabled() {
|
|
648
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
649
|
+
let config = Config {
|
|
650
|
+
failure_threshold: Some(1),
|
|
651
|
+
half_open_timeout_secs: 1.0, // 1 second timeout
|
|
652
|
+
jitter_factor: 0.0, // No jitter
|
|
653
|
+
..Default::default()
|
|
654
|
+
};
|
|
655
|
+
|
|
656
|
+
let ctx = CircuitContext {
|
|
657
|
+
name: "test_circuit".to_string(),
|
|
658
|
+
config,
|
|
659
|
+
storage: storage.clone(),
|
|
660
|
+
};
|
|
661
|
+
|
|
662
|
+
// Open circuit
|
|
663
|
+
storage.record_failure("test_circuit", 0.1);
|
|
664
|
+
let mut circuit = DynamicCircuit::new(ctx.clone());
|
|
665
|
+
circuit.handle(CircuitEvent::Trip).expect("Should open");
|
|
666
|
+
|
|
667
|
+
// Set opened_at
|
|
668
|
+
if let Some(data) = circuit.open_data_mut() {
|
|
669
|
+
data.opened_at = storage.monotonic_time();
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// Wait exactly 1 second
|
|
673
|
+
std::thread::sleep(std::time::Duration::from_secs(1));
|
|
674
|
+
|
|
675
|
+
// Should transition to HalfOpen (no jitter = exact timeout)
|
|
676
|
+
circuit
|
|
677
|
+
.handle(CircuitEvent::AttemptReset)
|
|
678
|
+
.expect("Should reset after exact timeout");
|
|
679
|
+
assert_eq!(circuit.current_state(), "HalfOpen");
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
#[test]
|
|
683
|
+
fn test_jitter_enabled() {
|
|
684
|
+
let storage = Arc::new(crate::MemoryStorage::new());
|
|
685
|
+
let config = Config {
|
|
686
|
+
failure_threshold: Some(1),
|
|
687
|
+
half_open_timeout_secs: 1.0,
|
|
688
|
+
jitter_factor: 0.1, // 10% jitter = 90-100% of timeout
|
|
689
|
+
..Default::default()
|
|
690
|
+
};
|
|
691
|
+
|
|
692
|
+
let ctx = CircuitContext {
|
|
693
|
+
name: "test_circuit".to_string(),
|
|
694
|
+
config,
|
|
695
|
+
storage: storage.clone(),
|
|
696
|
+
};
|
|
697
|
+
|
|
698
|
+
// Test multiple times to verify jitter reduces timeout
|
|
699
|
+
let mut found_early_reset = false;
|
|
700
|
+
for _ in 0..10 {
|
|
701
|
+
// Open circuit
|
|
702
|
+
storage.record_failure("test_circuit", 0.1);
|
|
703
|
+
let mut circuit = DynamicCircuit::new(ctx.clone());
|
|
704
|
+
circuit.handle(CircuitEvent::Trip).expect("Should open");
|
|
705
|
+
|
|
706
|
+
if let Some(data) = circuit.open_data_mut() {
|
|
707
|
+
data.opened_at = storage.monotonic_time();
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// With 10% jitter, timeout should be 900-1000ms
|
|
711
|
+
// Try at 950ms - should sometimes succeed (jitter applied)
|
|
712
|
+
std::thread::sleep(std::time::Duration::from_millis(950));
|
|
713
|
+
|
|
714
|
+
if circuit.handle(CircuitEvent::AttemptReset).is_ok() {
|
|
715
|
+
found_early_reset = true;
|
|
716
|
+
break;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
storage.clear("test_circuit");
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
assert!(
|
|
723
|
+
found_early_reset,
|
|
724
|
+
"Jitter should occasionally allow reset before full timeout"
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
#[test]
|
|
729
|
+
fn test_builder_with_jitter() {
|
|
730
|
+
let mut circuit = CircuitBreaker::builder("test")
|
|
731
|
+
.failure_threshold(2)
|
|
732
|
+
.half_open_timeout_secs(1.0)
|
|
733
|
+
.jitter_factor(0.5) // 50% jitter
|
|
734
|
+
.build();
|
|
735
|
+
|
|
736
|
+
// Trigger failures
|
|
737
|
+
let _ = circuit.call(|| Err::<(), _>("error 1"));
|
|
738
|
+
let _ = circuit.call(|| Err::<(), _>("error 2"));
|
|
739
|
+
assert!(circuit.is_open());
|
|
740
|
+
|
|
741
|
+
// Verify jitter_factor was set
|
|
742
|
+
assert_eq!(circuit.context.config.jitter_factor, 0.5);
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
#[test]
|
|
746
|
+
fn test_fallback_when_open() {
|
|
747
|
+
let mut circuit = CircuitBreaker::builder("test").failure_threshold(2).build();
|
|
748
|
+
|
|
749
|
+
// Trigger failures to open circuit
|
|
750
|
+
let _ = circuit.call(|| Err::<(), _>("error 1"));
|
|
751
|
+
let _ = circuit.call(|| Err::<(), _>("error 2"));
|
|
752
|
+
assert!(circuit.is_open());
|
|
753
|
+
|
|
754
|
+
// Call with fallback should return fallback result
|
|
755
|
+
let result = circuit.call((
|
|
756
|
+
|| Err::<String, _>("should not execute"),
|
|
757
|
+
CallOptions::new().with_fallback(|ctx| {
|
|
758
|
+
assert_eq!(ctx.circuit_name, "test");
|
|
759
|
+
assert_eq!(ctx.state, "Open");
|
|
760
|
+
Ok("fallback response".to_string())
|
|
761
|
+
}),
|
|
762
|
+
));
|
|
763
|
+
|
|
764
|
+
assert!(result.is_ok());
|
|
765
|
+
assert_eq!(result.unwrap(), "fallback response");
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
#[test]
|
|
769
|
+
fn test_fallback_error_propagation() {
|
|
770
|
+
let mut circuit = CircuitBreaker::builder("test").failure_threshold(1).build();
|
|
771
|
+
|
|
772
|
+
// Trigger failure to open circuit
|
|
773
|
+
let _ = circuit.call(|| Err::<(), _>("error"));
|
|
774
|
+
assert!(circuit.is_open());
|
|
775
|
+
|
|
776
|
+
// Fallback can also return errors
|
|
777
|
+
let result = circuit.call((
|
|
778
|
+
|| Ok::<String, _>("should not execute".to_string()),
|
|
779
|
+
CallOptions::new().with_fallback(|_ctx| Err::<String, _>("fallback error")),
|
|
780
|
+
));
|
|
781
|
+
|
|
782
|
+
assert!(result.is_err());
|
|
783
|
+
match result {
|
|
784
|
+
Err(CircuitError::Execution(e)) => assert_eq!(e, "fallback error"),
|
|
785
|
+
_ => panic!("Expected CircuitError::Execution"),
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
#[test]
|
|
790
|
+
fn test_rate_based_threshold() {
|
|
791
|
+
let mut circuit = CircuitBreaker::builder("test")
|
|
792
|
+
.disable_failure_threshold() // Only use rate-based
|
|
793
|
+
.failure_rate(0.5) // 50% failure rate
|
|
794
|
+
.minimum_calls(10)
|
|
795
|
+
.build();
|
|
796
|
+
|
|
797
|
+
// First 9 calls - below minimum, circuit stays closed
|
|
798
|
+
for i in 0..9 {
|
|
799
|
+
let _result = if i % 2 == 0 {
|
|
800
|
+
circuit.call(|| Ok::<(), _>(()))
|
|
801
|
+
} else {
|
|
802
|
+
circuit.call(|| Err::<(), _>("error"))
|
|
803
|
+
};
|
|
804
|
+
// Even with failures, circuit should stay closed (below minimum calls)
|
|
805
|
+
assert!(circuit.is_closed(), "Circuit opened before minimum calls");
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
// 10th call - now at minimum, with 5 failures out of 10 = 50% rate
|
|
809
|
+
// This should trip the circuit
|
|
810
|
+
let _ = circuit.call(|| Err::<(), _>("error"));
|
|
811
|
+
|
|
812
|
+
// Circuit should now be open (failure rate reached threshold)
|
|
813
|
+
assert!(circuit.is_open(), "Circuit did not open at rate threshold");
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
#[test]
|
|
817
|
+
fn test_rate_and_absolute_threshold_both_active() {
|
|
818
|
+
let mut circuit = CircuitBreaker::builder("test")
|
|
819
|
+
.failure_threshold(3) // Absolute: 3 failures
|
|
820
|
+
.failure_rate(0.8) // Rate: 80%
|
|
821
|
+
.minimum_calls(10)
|
|
822
|
+
.build();
|
|
823
|
+
|
|
824
|
+
// Trigger 3 failures quickly - should open via absolute threshold
|
|
825
|
+
let _ = circuit.call(|| Err::<(), _>("error 1"));
|
|
826
|
+
let _ = circuit.call(|| Err::<(), _>("error 2"));
|
|
827
|
+
assert!(circuit.is_closed());
|
|
828
|
+
|
|
829
|
+
let _ = circuit.call(|| Err::<(), _>("error 3"));
|
|
830
|
+
assert!(
|
|
831
|
+
circuit.is_open(),
|
|
832
|
+
"Circuit did not open at absolute threshold"
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
#[test]
|
|
837
|
+
fn test_minimum_calls_prevents_premature_trip() {
|
|
838
|
+
let mut circuit = CircuitBreaker::builder("test")
|
|
839
|
+
.disable_failure_threshold()
|
|
840
|
+
.failure_rate(0.5)
|
|
841
|
+
.minimum_calls(20)
|
|
842
|
+
.build();
|
|
843
|
+
|
|
844
|
+
// Record 10 failures out of 10 calls = 100% failure rate
|
|
845
|
+
for _ in 0..10 {
|
|
846
|
+
let _ = circuit.call(|| Err::<(), _>("error"));
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
// Circuit should still be closed (below minimum_calls)
|
|
850
|
+
assert!(
|
|
851
|
+
circuit.is_closed(),
|
|
852
|
+
"Circuit opened before reaching minimum_calls"
|
|
853
|
+
);
|
|
854
|
+
}
|
|
855
|
+
}
|