breaker_machines 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +25 -3
  3. data/ext/breaker_machines_native/Cargo.toml +8 -0
  4. data/ext/breaker_machines_native/core/Cargo.toml +18 -0
  5. data/ext/breaker_machines_native/core/examples/basic.rs +61 -0
  6. data/ext/breaker_machines_native/core/src/builder.rs +232 -0
  7. data/ext/breaker_machines_native/core/src/bulkhead.rs +223 -0
  8. data/ext/breaker_machines_native/core/src/callbacks.rs +58 -0
  9. data/ext/breaker_machines_native/core/src/circuit.rs +1156 -0
  10. data/ext/breaker_machines_native/core/src/classifier.rs +177 -0
  11. data/ext/breaker_machines_native/core/src/errors.rs +47 -0
  12. data/ext/breaker_machines_native/core/src/lib.rs +62 -0
  13. data/ext/breaker_machines_native/core/src/storage.rs +377 -0
  14. data/ext/breaker_machines_native/extconf.rb +40 -0
  15. data/ext/breaker_machines_native/ffi/Cargo.toml +16 -0
  16. data/ext/breaker_machines_native/ffi/src/lib.rs +218 -0
  17. data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/common.rs +355 -0
  18. data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/dynamic.rs +276 -0
  19. data/ext/breaker_machines_native/target/debug/build/clang-sys-d961dfabd5f43fba/out/macros.rs +49 -0
  20. data/ext/breaker_machines_native/target/debug/build/rb-sys-2bb7281aac8faec8/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
  21. data/ext/breaker_machines_native/target/debug/build/rb-sys-54cb99ea6aeab8bc/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
  22. data/ext/breaker_machines_native/target/debug/build/rb-sys-9e64a270c6421e93/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
  23. data/ext/breaker_machines_native/target/debug/build/rb-sys-e627030114d3fc19/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
  24. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/Cargo.toml +48 -0
  25. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/examples/basic.rs +61 -0
  26. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/builder.rs +154 -0
  27. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/callbacks.rs +55 -0
  28. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/circuit.rs +607 -0
  29. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/errors.rs +38 -0
  30. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/lib.rs +58 -0
  31. data/ext/breaker_machines_native/target/package/breaker-machines-0.1.0/src/storage.rs +377 -0
  32. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/Cargo.toml +48 -0
  33. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/examples/basic.rs +61 -0
  34. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/builder.rs +173 -0
  35. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/callbacks.rs +55 -0
  36. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/circuit.rs +855 -0
  37. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/errors.rs +38 -0
  38. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/lib.rs +58 -0
  39. data/ext/breaker_machines_native/target/package/breaker-machines-0.2.0/src/storage.rs +377 -0
  40. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/Cargo.toml +48 -0
  41. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/examples/basic.rs +61 -0
  42. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/builder.rs +154 -0
  43. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/callbacks.rs +55 -0
  44. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/circuit.rs +607 -0
  45. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/errors.rs +38 -0
  46. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/lib.rs +58 -0
  47. data/ext/breaker_machines_native/target/package/breaker-machines-0.5.0/src/storage.rs +377 -0
  48. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/Cargo.toml +48 -0
  49. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/examples/basic.rs +61 -0
  50. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/builder.rs +232 -0
  51. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/bulkhead.rs +223 -0
  52. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/callbacks.rs +58 -0
  53. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/circuit.rs +1156 -0
  54. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/classifier.rs +177 -0
  55. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/errors.rs +47 -0
  56. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/lib.rs +62 -0
  57. data/ext/breaker_machines_native/target/package/breaker-machines-0.6.0/src/storage.rs +377 -0
  58. data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/common.rs +355 -0
  59. data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/dynamic.rs +276 -0
  60. data/ext/breaker_machines_native/target/release/build/clang-sys-ef8ad8b846ac8b75/out/macros.rs +49 -0
  61. data/ext/breaker_machines_native/target/release/build/rb-sys-064bf9961dd17810/out/bindings-0.9.117-mri-arm64-darwin24-3.4.7.rs +8936 -0
  62. data/lib/breaker_machines/async_circuit.rb +47 -0
  63. data/lib/breaker_machines/async_support.rb +4 -3
  64. data/lib/breaker_machines/cascading_circuit.rb +5 -3
  65. data/lib/breaker_machines/circuit/async_state_management.rb +71 -0
  66. data/lib/breaker_machines/circuit/base.rb +59 -0
  67. data/lib/breaker_machines/circuit/callbacks.rb +7 -12
  68. data/lib/breaker_machines/circuit/configuration.rb +6 -26
  69. data/lib/breaker_machines/circuit/coordinated_state_management.rb +117 -0
  70. data/lib/breaker_machines/circuit/hedged_execution.rb +115 -0
  71. data/lib/breaker_machines/circuit/introspection.rb +1 -0
  72. data/lib/breaker_machines/circuit/native.rb +127 -0
  73. data/lib/breaker_machines/circuit/state_callbacks.rb +72 -0
  74. data/lib/breaker_machines/circuit/state_management.rb +14 -61
  75. data/lib/breaker_machines/circuit.rb +1 -7
  76. data/lib/breaker_machines/circuit_group.rb +153 -0
  77. data/lib/breaker_machines/coordinated_circuit.rb +10 -0
  78. data/lib/breaker_machines/dsl.rb +2 -2
  79. data/lib/breaker_machines/errors.rb +20 -0
  80. data/lib/breaker_machines/hedged_async_support.rb +29 -36
  81. data/lib/breaker_machines/native_extension.rb +36 -0
  82. data/lib/breaker_machines/native_speedup.rb +6 -0
  83. data/lib/breaker_machines/storage/bucket_memory.rb +4 -1
  84. data/lib/breaker_machines/storage/memory.rb +4 -1
  85. data/lib/breaker_machines/storage/native.rb +90 -0
  86. data/lib/breaker_machines/version.rb +1 -1
  87. data/lib/breaker_machines.rb +115 -11
  88. data/lib/breaker_machines_native/breaker_machines_native.bundle +0 -0
  89. data/sig/breaker_machines.rbs +20 -8
  90. metadata +107 -7
  91. data/lib/breaker_machines/hedged_execution.rb +0 -113
@@ -0,0 +1,855 @@
1
+ //! Circuit breaker implementation using state machines
2
+ //!
3
+ //! This module provides a complete circuit breaker with state management.
4
+
5
+ use crate::{callbacks::Callbacks, errors::CircuitError, StorageBackend};
6
+ use state_machines::state_machine;
7
+ use std::sync::Arc;
8
+
9
+ /// Circuit breaker configuration
10
+ #[derive(Debug, Clone)]
11
+ pub struct Config {
12
+ /// Number of failures required to open the circuit (absolute count)
13
+ /// If None, only rate-based threshold is used
14
+ pub failure_threshold: Option<usize>,
15
+
16
+ /// Failure rate threshold (0.0-1.0) - percentage of failures to open circuit
17
+ /// If None, only absolute count threshold is used
18
+ pub failure_rate_threshold: Option<f64>,
19
+
20
+ /// Minimum number of calls before rate-based threshold is evaluated
21
+ pub minimum_calls: usize,
22
+
23
+ /// Time window in seconds for counting failures
24
+ pub failure_window_secs: f64,
25
+
26
+ /// Timeout in seconds before transitioning from Open to HalfOpen
27
+ pub half_open_timeout_secs: f64,
28
+
29
+ /// Number of successes required in HalfOpen to close the circuit
30
+ pub success_threshold: usize,
31
+
32
+ /// Jitter factor for half_open_timeout (0.0 = no jitter, 1.0 = full jitter)
33
+ /// Uses chrono-machines formula: timeout * (1 - jitter + rand * jitter)
34
+ pub jitter_factor: f64,
35
+ }
36
+
37
+ impl Default for Config {
38
+ fn default() -> Self {
39
+ Self {
40
+ failure_threshold: Some(5),
41
+ failure_rate_threshold: None,
42
+ minimum_calls: 20,
43
+ failure_window_secs: 60.0,
44
+ half_open_timeout_secs: 30.0,
45
+ success_threshold: 2,
46
+ jitter_factor: 0.0,
47
+ }
48
+ }
49
+ }
50
+
51
+ /// Context provided to fallback closures when circuit is open
52
+ #[derive(Debug, Clone)]
53
+ pub struct FallbackContext {
54
+ /// Circuit name
55
+ pub circuit_name: String,
56
+ /// Timestamp when circuit opened
57
+ pub opened_at: f64,
58
+ /// Current circuit state
59
+ pub state: &'static str,
60
+ }
61
+
62
+ /// Options for circuit breaker calls
63
+ pub struct CallOptions<T, E> {
64
+ /// Optional fallback function called when circuit is open
65
+ pub fallback: Option<Box<dyn FnOnce(&FallbackContext) -> Result<T, E> + Send>>,
66
+ }
67
+
68
+ impl<T, E> Default for CallOptions<T, E> {
69
+ fn default() -> Self {
70
+ Self { fallback: None }
71
+ }
72
+ }
73
+
74
+ impl<T, E> CallOptions<T, E> {
75
+ /// Create new call options with no fallback
76
+ pub fn new() -> Self {
77
+ Self::default()
78
+ }
79
+
80
+ /// Set a fallback function
81
+ pub fn with_fallback<F>(mut self, f: F) -> Self
82
+ where
83
+ F: FnOnce(&FallbackContext) -> Result<T, E> + Send + 'static,
84
+ {
85
+ self.fallback = Some(Box::new(f));
86
+ self
87
+ }
88
+ }
89
+
90
+ /// Trait for converting into CallOptions - allows flexible call() API
91
+ pub trait IntoCallOptions<T, E> {
92
+ fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>);
93
+ }
94
+
95
+ /// Implement for plain closures (backward compatibility)
96
+ impl<T, E, F> IntoCallOptions<T, E> for F
97
+ where
98
+ F: FnOnce() -> Result<T, E> + 'static,
99
+ {
100
+ fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>) {
101
+ (Box::new(self), CallOptions::default())
102
+ }
103
+ }
104
+
105
+ /// Implement for (closure, CallOptions) tuple
106
+ impl<T, E, F> IntoCallOptions<T, E> for (F, CallOptions<T, E>)
107
+ where
108
+ F: FnOnce() -> Result<T, E> + 'static,
109
+ {
110
+ fn into_call_options(self) -> (Box<dyn FnOnce() -> Result<T, E>>, CallOptions<T, E>) {
111
+ (Box::new(self.0), self.1)
112
+ }
113
+ }
114
+
115
+ /// Circuit breaker context - shared data across all states
116
+ #[derive(Clone)]
117
+ pub struct CircuitContext {
118
+ pub name: String,
119
+ pub config: Config,
120
+ pub storage: Arc<dyn StorageBackend>,
121
+ }
122
+
123
+ impl Default for CircuitContext {
124
+ fn default() -> Self {
125
+ Self {
126
+ name: String::new(),
127
+ config: Config::default(),
128
+ storage: Arc::new(crate::MemoryStorage::new()),
129
+ }
130
+ }
131
+ }
132
+
133
+ impl std::fmt::Debug for CircuitContext {
134
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
135
+ f.debug_struct("CircuitContext")
136
+ .field("name", &self.name)
137
+ .field("config", &self.config)
138
+ .field("storage", &"<dyn StorageBackend>")
139
+ .finish()
140
+ }
141
+ }
142
+
143
+ /// Data specific to the Open state
144
+ #[derive(Debug, Clone, Default)]
145
+ pub struct OpenData {
146
+ pub opened_at: f64,
147
+ }
148
+
149
+ /// Data specific to the HalfOpen state
150
+ #[derive(Debug, Clone, Default)]
151
+ pub struct HalfOpenData {
152
+ pub consecutive_successes: usize,
153
+ }
154
+
155
+ // Define the circuit breaker state machine with dynamic mode
156
+ state_machine! {
157
+ name: Circuit,
158
+ context: CircuitContext,
159
+ dynamic: true, // Enable dynamic mode for runtime state transitions
160
+
161
+ initial: Closed,
162
+ states: [
163
+ Closed,
164
+ Open(OpenData),
165
+ HalfOpen(HalfOpenData),
166
+ ],
167
+ events {
168
+ trip {
169
+ guards: [should_open],
170
+ transition: { from: [Closed, HalfOpen], to: Open }
171
+ }
172
+ attempt_reset {
173
+ guards: [timeout_elapsed],
174
+ transition: { from: Open, to: HalfOpen }
175
+ }
176
+ close {
177
+ guards: [should_close],
178
+ transition: { from: HalfOpen, to: Closed }
179
+ }
180
+ }
181
+ }
182
+
183
+ // Guards for dynamic mode - implemented on typestate machines
184
+ impl Circuit<Closed> {
185
+ /// Check if failure threshold is exceeded (absolute count or rate-based)
186
+ fn should_open(&self, ctx: &CircuitContext) -> bool {
187
+ let failures = ctx
188
+ .storage
189
+ .failure_count(&ctx.name, ctx.config.failure_window_secs);
190
+
191
+ // Check absolute count threshold
192
+ if let Some(threshold) = ctx.config.failure_threshold {
193
+ if failures >= threshold {
194
+ return true;
195
+ }
196
+ }
197
+
198
+ // Check rate-based threshold
199
+ if let Some(rate_threshold) = ctx.config.failure_rate_threshold {
200
+ let successes = ctx
201
+ .storage
202
+ .success_count(&ctx.name, ctx.config.failure_window_secs);
203
+ let total = failures + successes;
204
+
205
+ // Only evaluate rate if we have minimum calls
206
+ if total >= ctx.config.minimum_calls {
207
+ let failure_rate = if total > 0 {
208
+ failures as f64 / total as f64
209
+ } else {
210
+ 0.0
211
+ };
212
+
213
+ if failure_rate >= rate_threshold {
214
+ return true;
215
+ }
216
+ }
217
+ }
218
+
219
+ false
220
+ }
221
+ }
222
+
223
+ impl Circuit<HalfOpen> {
224
+ /// Check if failure threshold is exceeded (absolute count or rate-based)
225
+ fn should_open(&self, ctx: &CircuitContext) -> bool {
226
+ let failures = ctx
227
+ .storage
228
+ .failure_count(&ctx.name, ctx.config.failure_window_secs);
229
+
230
+ // Check absolute count threshold
231
+ if let Some(threshold) = ctx.config.failure_threshold {
232
+ if failures >= threshold {
233
+ return true;
234
+ }
235
+ }
236
+
237
+ // Check rate-based threshold
238
+ if let Some(rate_threshold) = ctx.config.failure_rate_threshold {
239
+ let successes = ctx
240
+ .storage
241
+ .success_count(&ctx.name, ctx.config.failure_window_secs);
242
+ let total = failures + successes;
243
+
244
+ // Only evaluate rate if we have minimum calls
245
+ if total >= ctx.config.minimum_calls {
246
+ let failure_rate = if total > 0 {
247
+ failures as f64 / total as f64
248
+ } else {
249
+ 0.0
250
+ };
251
+
252
+ if failure_rate >= rate_threshold {
253
+ return true;
254
+ }
255
+ }
256
+ }
257
+
258
+ false
259
+ }
260
+
261
+ /// Check if enough successes to close circuit
262
+ fn should_close(&self, ctx: &CircuitContext) -> bool {
263
+ let data = self
264
+ .state_data_half_open()
265
+ .expect("HalfOpen state must have data");
266
+ data.consecutive_successes >= ctx.config.success_threshold
267
+ }
268
+ }
269
+
270
+ impl Circuit<Open> {
271
+ /// Check if timeout has elapsed for Open -> HalfOpen transition
272
+ fn timeout_elapsed(&self, ctx: &CircuitContext) -> bool {
273
+ let data = self.state_data_open().expect("Open state must have data");
274
+ let current_time = ctx.storage.monotonic_time();
275
+ let elapsed = current_time - data.opened_at;
276
+
277
+ // Apply jitter using chrono-machines if jitter_factor > 0
278
+ let timeout_secs = if ctx.config.jitter_factor > 0.0 {
279
+ let policy = chrono_machines::Policy {
280
+ max_attempts: 1,
281
+ base_delay_ms: (ctx.config.half_open_timeout_secs * 1000.0) as u64,
282
+ multiplier: 1.0,
283
+ max_delay_ms: (ctx.config.half_open_timeout_secs * 1000.0) as u64,
284
+ };
285
+ let timeout_ms = policy.calculate_delay(1, ctx.config.jitter_factor);
286
+ (timeout_ms as f64) / 1000.0
287
+ } else {
288
+ ctx.config.half_open_timeout_secs
289
+ };
290
+
291
+ elapsed >= timeout_secs
292
+ }
293
+ }
294
+
295
+ /// Circuit breaker public API
296
+ pub struct CircuitBreaker {
297
+ machine: DynamicCircuit,
298
+ context: CircuitContext,
299
+ callbacks: Callbacks,
300
+ }
301
+
302
+ impl CircuitBreaker {
303
+ /// Create a new circuit breaker (use builder() for more options)
304
+ pub fn new(name: String, config: Config) -> Self {
305
+ let storage = Arc::new(crate::MemoryStorage::new());
306
+ let context = CircuitContext {
307
+ name,
308
+ config,
309
+ storage,
310
+ };
311
+
312
+ let machine = DynamicCircuit::new(context.clone());
313
+ let callbacks = Callbacks::new();
314
+
315
+ Self {
316
+ machine,
317
+ context,
318
+ callbacks,
319
+ }
320
+ }
321
+
322
+ /// Create a circuit breaker with custom context and callbacks (used by builder)
323
+ pub(crate) fn with_context_and_callbacks(
324
+ context: CircuitContext,
325
+ callbacks: Callbacks,
326
+ ) -> Self {
327
+ let machine = DynamicCircuit::new(context.clone());
328
+
329
+ Self {
330
+ machine,
331
+ context,
332
+ callbacks,
333
+ }
334
+ }
335
+
336
+ /// Create a new circuit breaker builder
337
+ pub fn builder(name: impl Into<String>) -> crate::builder::CircuitBuilder {
338
+ crate::builder::CircuitBuilder::new(name)
339
+ }
340
+
341
+ /// Execute a fallible operation with circuit breaker protection
342
+ ///
343
+ /// Accepts either:
344
+ /// - A plain closure: `circuit.call(|| api_request())`
345
+ /// - A closure with options: `circuit.call((|| api_request(), CallOptions::new().with_fallback(...)))`
346
+ pub fn call<I, T, E>(&mut self, input: I) -> Result<T, CircuitError<E>>
347
+ where
348
+ I: IntoCallOptions<T, E>,
349
+ {
350
+ let (f, options) = input.into_call_options();
351
+
352
+ // Check for timeout-based Open -> HalfOpen transition
353
+ if self.machine.current_state() == "Open" {
354
+ let _ = self.machine.handle(CircuitEvent::AttemptReset);
355
+ if self.machine.current_state() == "HalfOpen" {
356
+ self.callbacks.trigger_half_open(&self.context.name);
357
+ }
358
+ }
359
+
360
+ // Handle based on current state
361
+ match self.machine.current_state() {
362
+ "Open" => {
363
+ let opened_at = self.machine.open_data().map(|d| d.opened_at).unwrap_or(0.0);
364
+
365
+ // If fallback is provided, use it instead of returning error
366
+ if let Some(fallback) = options.fallback {
367
+ let ctx = FallbackContext {
368
+ circuit_name: self.context.name.clone(),
369
+ opened_at,
370
+ state: "Open",
371
+ };
372
+ return fallback(&ctx).map_err(CircuitError::Execution);
373
+ }
374
+
375
+ Err(CircuitError::Open {
376
+ circuit: self.context.name.clone(),
377
+ opened_at,
378
+ })
379
+ }
380
+ "HalfOpen" => {
381
+ // Check if we've reached the success threshold
382
+ if let Some(data) = self.machine.half_open_data() {
383
+ if data.consecutive_successes >= self.context.config.success_threshold {
384
+ return Err(CircuitError::HalfOpenLimitReached {
385
+ circuit: self.context.name.clone(),
386
+ });
387
+ }
388
+ }
389
+ self.execute_call(f)
390
+ }
391
+ _ => self.execute_call(f),
392
+ }
393
+ }
394
+
395
+ fn execute_call<T, E>(
396
+ &mut self,
397
+ f: Box<dyn FnOnce() -> Result<T, E>>,
398
+ ) -> Result<T, CircuitError<E>> {
399
+ let start = self.context.storage.monotonic_time();
400
+
401
+ match f() {
402
+ Ok(val) => {
403
+ let duration = self.context.storage.monotonic_time() - start;
404
+ self.context
405
+ .storage
406
+ .record_success(&self.context.name, duration);
407
+
408
+ // Handle success in HalfOpen state
409
+ if self.machine.current_state() == "HalfOpen" {
410
+ if let Some(data) = self.machine.half_open_data_mut() {
411
+ data.consecutive_successes += 1;
412
+ }
413
+
414
+ // Try to close the circuit
415
+ if self.machine.handle(CircuitEvent::Close).is_ok() {
416
+ self.callbacks.trigger_close(&self.context.name);
417
+ }
418
+ }
419
+
420
+ Ok(val)
421
+ }
422
+ Err(e) => {
423
+ let duration = self.context.storage.monotonic_time() - start;
424
+ self.context
425
+ .storage
426
+ .record_failure(&self.context.name, duration);
427
+
428
+ // Try to trip the circuit
429
+ let result = self.machine.handle(CircuitEvent::Trip);
430
+ if result.is_ok() {
431
+ // Transition succeeded - update opened_at timestamp
432
+ if let Some(data) = self.machine.open_data_mut() {
433
+ data.opened_at = self.context.storage.monotonic_time();
434
+ }
435
+ self.callbacks.trigger_open(&self.context.name);
436
+ }
437
+
438
+ Err(CircuitError::Execution(e))
439
+ }
440
+ }
441
+ }
442
+
443
+ /// Record a successful operation (for manual tracking)
444
+ pub fn record_success(&self, duration: f64) {
445
+ self.context
446
+ .storage
447
+ .record_success(&self.context.name, duration);
448
+ }
449
+
450
+ /// Record a failed operation (for manual tracking)
451
+ pub fn record_failure(&self, duration: f64) {
452
+ self.context
453
+ .storage
454
+ .record_failure(&self.context.name, duration);
455
+ }
456
+
457
+ /// Check if circuit is open
458
+ pub fn is_open(&self) -> bool {
459
+ self.machine.current_state() == "Open"
460
+ }
461
+
462
+ /// Check if circuit is closed
463
+ pub fn is_closed(&self) -> bool {
464
+ self.machine.current_state() == "Closed"
465
+ }
466
+
467
+ /// Get current state name
468
+ pub fn state_name(&self) -> &'static str {
469
+ self.machine.current_state()
470
+ }
471
+
472
+ /// Clear all events and reset circuit to Closed state
473
+ pub fn reset(&mut self) {
474
+ self.context.storage.clear(&self.context.name);
475
+ // Recreate machine in Closed state
476
+ self.machine = DynamicCircuit::new(self.context.clone());
477
+ }
478
+ }
479
+
480
+ #[cfg(test)]
481
+ mod tests {
482
+ use super::*;
483
+
484
+ #[test]
485
+ fn test_circuit_breaker_creation() {
486
+ let config = Config::default();
487
+ let circuit = CircuitBreaker::new("test".to_string(), config);
488
+
489
+ assert!(circuit.is_closed());
490
+ assert!(!circuit.is_open());
491
+ }
492
+
493
+ #[test]
494
+ fn test_circuit_opens_after_threshold() {
495
+ let config = Config {
496
+ failure_threshold: Some(3),
497
+ ..Default::default()
498
+ };
499
+
500
+ let mut circuit = CircuitBreaker::new("test".to_string(), config);
501
+
502
+ // Trigger failures via call() method
503
+ let _ = circuit.call(|| Err::<(), _>("error 1"));
504
+ let _ = circuit.call(|| Err::<(), _>("error 2"));
505
+ assert!(circuit.is_closed());
506
+
507
+ let _ = circuit.call(|| Err::<(), _>("error 3"));
508
+ assert!(circuit.is_open());
509
+ }
510
+
511
+ #[test]
512
+ fn test_reset_clears_state() {
513
+ let config = Config {
514
+ failure_threshold: Some(2),
515
+ ..Default::default()
516
+ };
517
+
518
+ let mut circuit = CircuitBreaker::new("test".to_string(), config);
519
+
520
+ // Trigger failures
521
+ let _ = circuit.call(|| Err::<(), _>("error 1"));
522
+ let _ = circuit.call(|| Err::<(), _>("error 2"));
523
+ assert!(circuit.is_open());
524
+
525
+ circuit.reset();
526
+ assert!(circuit.is_closed());
527
+ }
528
+
529
+ #[test]
530
+ fn test_state_machine_closed_to_open_transition() {
531
+ let storage = Arc::new(crate::MemoryStorage::new());
532
+ let config = Config {
533
+ failure_threshold: Some(3),
534
+ ..Default::default()
535
+ };
536
+
537
+ let ctx = CircuitContext {
538
+ name: "test_circuit".to_string(),
539
+ config,
540
+ storage: storage.clone(),
541
+ };
542
+
543
+ let mut circuit = DynamicCircuit::new(ctx.clone());
544
+
545
+ // Initially closed - trip should fail guard
546
+ let result = circuit.handle(CircuitEvent::Trip);
547
+ assert!(result.is_err(), "Should fail guard when below threshold");
548
+
549
+ // Record failures to exceed threshold
550
+ storage.record_failure("test_circuit", 0.1);
551
+ storage.record_failure("test_circuit", 0.1);
552
+ storage.record_failure("test_circuit", 0.1);
553
+
554
+ // Now trip should succeed - guards pass
555
+ circuit
556
+ .handle(CircuitEvent::Trip)
557
+ .expect("Should open after reaching threshold");
558
+
559
+ assert_eq!(circuit.current_state(), "Open");
560
+ }
561
+
562
+ #[test]
563
+ fn test_state_machine_open_to_half_open_transition() {
564
+ let storage = Arc::new(crate::MemoryStorage::new());
565
+ let config = Config {
566
+ failure_threshold: Some(2),
567
+ half_open_timeout_secs: 0.001, // Very short timeout for testing
568
+ ..Default::default()
569
+ };
570
+
571
+ let ctx = CircuitContext {
572
+ name: "test_circuit".to_string(),
573
+ config,
574
+ storage: storage.clone(),
575
+ };
576
+
577
+ // Record failures and open circuit
578
+ storage.record_failure("test_circuit", 0.1);
579
+ storage.record_failure("test_circuit", 0.1);
580
+
581
+ let mut circuit = DynamicCircuit::new(ctx.clone());
582
+ circuit.handle(CircuitEvent::Trip).expect("Should open");
583
+
584
+ // Set opened_at timestamp
585
+ if let Some(data) = circuit.open_data_mut() {
586
+ data.opened_at = storage.monotonic_time();
587
+ }
588
+
589
+ // Immediately try to reset - should fail guard (timeout not elapsed)
590
+ let result = circuit.handle(CircuitEvent::AttemptReset);
591
+ assert!(
592
+ result.is_err(),
593
+ "Should fail guard when timeout not elapsed"
594
+ );
595
+
596
+ // Wait for timeout
597
+ std::thread::sleep(std::time::Duration::from_millis(5));
598
+
599
+ circuit
600
+ .handle(CircuitEvent::AttemptReset)
601
+ .expect("Should reset after timeout");
602
+
603
+ // Verify we're in HalfOpen state
604
+ assert_eq!(circuit.current_state(), "HalfOpen");
605
+ let data = circuit.half_open_data().expect("Should have HalfOpen data");
606
+ assert_eq!(data.consecutive_successes, 0);
607
+ }
608
+
609
+ #[test]
610
+ fn test_state_machine_half_open_to_closed_guard() {
611
+ let storage = Arc::new(crate::MemoryStorage::new());
612
+ let config = Config {
613
+ failure_threshold: Some(2),
614
+ half_open_timeout_secs: 0.001,
615
+ ..Default::default()
616
+ };
617
+
618
+ let ctx = CircuitContext {
619
+ name: "test_circuit".to_string(),
620
+ config,
621
+ storage: storage.clone(),
622
+ };
623
+
624
+ // Get to HalfOpen state
625
+ storage.record_failure("test_circuit", 0.1);
626
+ storage.record_failure("test_circuit", 0.1);
627
+
628
+ let mut circuit = DynamicCircuit::new(ctx.clone());
629
+ circuit.handle(CircuitEvent::Trip).expect("Should open");
630
+
631
+ // Set opened_at and wait for timeout
632
+ if let Some(data) = circuit.open_data_mut() {
633
+ data.opened_at = storage.monotonic_time();
634
+ }
635
+ std::thread::sleep(std::time::Duration::from_millis(5));
636
+
637
+ circuit
638
+ .handle(CircuitEvent::AttemptReset)
639
+ .expect("Should reset");
640
+
641
+ // Try to close - should fail guard (not enough successes)
642
+ let result = circuit.handle(CircuitEvent::Close);
643
+ assert!(result.is_err(), "Should fail guard without successes");
644
+ }
645
+
646
+ #[test]
647
+ fn test_jitter_disabled() {
648
+ let storage = Arc::new(crate::MemoryStorage::new());
649
+ let config = Config {
650
+ failure_threshold: Some(1),
651
+ half_open_timeout_secs: 1.0, // 1 second timeout
652
+ jitter_factor: 0.0, // No jitter
653
+ ..Default::default()
654
+ };
655
+
656
+ let ctx = CircuitContext {
657
+ name: "test_circuit".to_string(),
658
+ config,
659
+ storage: storage.clone(),
660
+ };
661
+
662
+ // Open circuit
663
+ storage.record_failure("test_circuit", 0.1);
664
+ let mut circuit = DynamicCircuit::new(ctx.clone());
665
+ circuit.handle(CircuitEvent::Trip).expect("Should open");
666
+
667
+ // Set opened_at
668
+ if let Some(data) = circuit.open_data_mut() {
669
+ data.opened_at = storage.monotonic_time();
670
+ }
671
+
672
+ // Wait exactly 1 second
673
+ std::thread::sleep(std::time::Duration::from_secs(1));
674
+
675
+ // Should transition to HalfOpen (no jitter = exact timeout)
676
+ circuit
677
+ .handle(CircuitEvent::AttemptReset)
678
+ .expect("Should reset after exact timeout");
679
+ assert_eq!(circuit.current_state(), "HalfOpen");
680
+ }
681
+
682
+ #[test]
683
+ fn test_jitter_enabled() {
684
+ let storage = Arc::new(crate::MemoryStorage::new());
685
+ let config = Config {
686
+ failure_threshold: Some(1),
687
+ half_open_timeout_secs: 1.0,
688
+ jitter_factor: 0.1, // 10% jitter = 90-100% of timeout
689
+ ..Default::default()
690
+ };
691
+
692
+ let ctx = CircuitContext {
693
+ name: "test_circuit".to_string(),
694
+ config,
695
+ storage: storage.clone(),
696
+ };
697
+
698
+ // Test multiple times to verify jitter reduces timeout
699
+ let mut found_early_reset = false;
700
+ for _ in 0..10 {
701
+ // Open circuit
702
+ storage.record_failure("test_circuit", 0.1);
703
+ let mut circuit = DynamicCircuit::new(ctx.clone());
704
+ circuit.handle(CircuitEvent::Trip).expect("Should open");
705
+
706
+ if let Some(data) = circuit.open_data_mut() {
707
+ data.opened_at = storage.monotonic_time();
708
+ }
709
+
710
+ // With 10% jitter, timeout should be 900-1000ms
711
+ // Try at 950ms - should sometimes succeed (jitter applied)
712
+ std::thread::sleep(std::time::Duration::from_millis(950));
713
+
714
+ if circuit.handle(CircuitEvent::AttemptReset).is_ok() {
715
+ found_early_reset = true;
716
+ break;
717
+ }
718
+
719
+ storage.clear("test_circuit");
720
+ }
721
+
722
+ assert!(
723
+ found_early_reset,
724
+ "Jitter should occasionally allow reset before full timeout"
725
+ );
726
+ }
727
+
728
+ #[test]
729
+ fn test_builder_with_jitter() {
730
+ let mut circuit = CircuitBreaker::builder("test")
731
+ .failure_threshold(2)
732
+ .half_open_timeout_secs(1.0)
733
+ .jitter_factor(0.5) // 50% jitter
734
+ .build();
735
+
736
+ // Trigger failures
737
+ let _ = circuit.call(|| Err::<(), _>("error 1"));
738
+ let _ = circuit.call(|| Err::<(), _>("error 2"));
739
+ assert!(circuit.is_open());
740
+
741
+ // Verify jitter_factor was set
742
+ assert_eq!(circuit.context.config.jitter_factor, 0.5);
743
+ }
744
+
745
+ #[test]
746
+ fn test_fallback_when_open() {
747
+ let mut circuit = CircuitBreaker::builder("test").failure_threshold(2).build();
748
+
749
+ // Trigger failures to open circuit
750
+ let _ = circuit.call(|| Err::<(), _>("error 1"));
751
+ let _ = circuit.call(|| Err::<(), _>("error 2"));
752
+ assert!(circuit.is_open());
753
+
754
+ // Call with fallback should return fallback result
755
+ let result = circuit.call((
756
+ || Err::<String, _>("should not execute"),
757
+ CallOptions::new().with_fallback(|ctx| {
758
+ assert_eq!(ctx.circuit_name, "test");
759
+ assert_eq!(ctx.state, "Open");
760
+ Ok("fallback response".to_string())
761
+ }),
762
+ ));
763
+
764
+ assert!(result.is_ok());
765
+ assert_eq!(result.unwrap(), "fallback response");
766
+ }
767
+
768
+ #[test]
769
+ fn test_fallback_error_propagation() {
770
+ let mut circuit = CircuitBreaker::builder("test").failure_threshold(1).build();
771
+
772
+ // Trigger failure to open circuit
773
+ let _ = circuit.call(|| Err::<(), _>("error"));
774
+ assert!(circuit.is_open());
775
+
776
+ // Fallback can also return errors
777
+ let result = circuit.call((
778
+ || Ok::<String, _>("should not execute".to_string()),
779
+ CallOptions::new().with_fallback(|_ctx| Err::<String, _>("fallback error")),
780
+ ));
781
+
782
+ assert!(result.is_err());
783
+ match result {
784
+ Err(CircuitError::Execution(e)) => assert_eq!(e, "fallback error"),
785
+ _ => panic!("Expected CircuitError::Execution"),
786
+ }
787
+ }
788
+
789
+ #[test]
790
+ fn test_rate_based_threshold() {
791
+ let mut circuit = CircuitBreaker::builder("test")
792
+ .disable_failure_threshold() // Only use rate-based
793
+ .failure_rate(0.5) // 50% failure rate
794
+ .minimum_calls(10)
795
+ .build();
796
+
797
+ // First 9 calls - below minimum, circuit stays closed
798
+ for i in 0..9 {
799
+ let _result = if i % 2 == 0 {
800
+ circuit.call(|| Ok::<(), _>(()))
801
+ } else {
802
+ circuit.call(|| Err::<(), _>("error"))
803
+ };
804
+ // Even with failures, circuit should stay closed (below minimum calls)
805
+ assert!(circuit.is_closed(), "Circuit opened before minimum calls");
806
+ }
807
+
808
+ // 10th call - now at minimum, with 5 failures out of 10 = 50% rate
809
+ // This should trip the circuit
810
+ let _ = circuit.call(|| Err::<(), _>("error"));
811
+
812
+ // Circuit should now be open (failure rate reached threshold)
813
+ assert!(circuit.is_open(), "Circuit did not open at rate threshold");
814
+ }
815
+
816
+ #[test]
817
+ fn test_rate_and_absolute_threshold_both_active() {
818
+ let mut circuit = CircuitBreaker::builder("test")
819
+ .failure_threshold(3) // Absolute: 3 failures
820
+ .failure_rate(0.8) // Rate: 80%
821
+ .minimum_calls(10)
822
+ .build();
823
+
824
+ // Trigger 3 failures quickly - should open via absolute threshold
825
+ let _ = circuit.call(|| Err::<(), _>("error 1"));
826
+ let _ = circuit.call(|| Err::<(), _>("error 2"));
827
+ assert!(circuit.is_closed());
828
+
829
+ let _ = circuit.call(|| Err::<(), _>("error 3"));
830
+ assert!(
831
+ circuit.is_open(),
832
+ "Circuit did not open at absolute threshold"
833
+ );
834
+ }
835
+
836
+ #[test]
837
+ fn test_minimum_calls_prevents_premature_trip() {
838
+ let mut circuit = CircuitBreaker::builder("test")
839
+ .disable_failure_threshold()
840
+ .failure_rate(0.5)
841
+ .minimum_calls(20)
842
+ .build();
843
+
844
+ // Record 10 failures out of 10 calls = 100% failure rate
845
+ for _ in 0..10 {
846
+ let _ = circuit.call(|| Err::<(), _>("error"));
847
+ }
848
+
849
+ // Circuit should still be closed (below minimum_calls)
850
+ assert!(
851
+ circuit.is_closed(),
852
+ "Circuit opened before reaching minimum_calls"
853
+ );
854
+ }
855
+ }