breaker_machines 0.9.2-x86_64-linux-musl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +184 -0
- data/ext/breaker_machines_native/extconf.rb +3 -0
- data/lib/breaker_machines/async_circuit.rb +47 -0
- data/lib/breaker_machines/async_support.rb +104 -0
- data/lib/breaker_machines/cascading_circuit.rb +177 -0
- data/lib/breaker_machines/circuit/async_state_management.rb +71 -0
- data/lib/breaker_machines/circuit/base.rb +59 -0
- data/lib/breaker_machines/circuit/callbacks.rb +135 -0
- data/lib/breaker_machines/circuit/configuration.rb +67 -0
- data/lib/breaker_machines/circuit/coordinated_state_management.rb +117 -0
- data/lib/breaker_machines/circuit/execution.rb +231 -0
- data/lib/breaker_machines/circuit/hedged_execution.rb +115 -0
- data/lib/breaker_machines/circuit/introspection.rb +93 -0
- data/lib/breaker_machines/circuit/native.rb +127 -0
- data/lib/breaker_machines/circuit/state_callbacks.rb +72 -0
- data/lib/breaker_machines/circuit/state_management.rb +59 -0
- data/lib/breaker_machines/circuit.rb +8 -0
- data/lib/breaker_machines/circuit_group.rb +153 -0
- data/lib/breaker_machines/console.rb +345 -0
- data/lib/breaker_machines/coordinated_circuit.rb +10 -0
- data/lib/breaker_machines/dsl/cascading_circuit_builder.rb +20 -0
- data/lib/breaker_machines/dsl/circuit_builder.rb +209 -0
- data/lib/breaker_machines/dsl/hedged_builder.rb +21 -0
- data/lib/breaker_machines/dsl/parallel_fallback_wrapper.rb +20 -0
- data/lib/breaker_machines/dsl.rb +283 -0
- data/lib/breaker_machines/errors.rb +71 -0
- data/lib/breaker_machines/hedged_async_support.rb +88 -0
- data/lib/breaker_machines/native_extension.rb +81 -0
- data/lib/breaker_machines/native_speedup.rb +10 -0
- data/lib/breaker_machines/registry.rb +243 -0
- data/lib/breaker_machines/storage/backend_state.rb +69 -0
- data/lib/breaker_machines/storage/base.rb +52 -0
- data/lib/breaker_machines/storage/bucket_memory.rb +176 -0
- data/lib/breaker_machines/storage/cache.rb +169 -0
- data/lib/breaker_machines/storage/fallback_chain.rb +294 -0
- data/lib/breaker_machines/storage/memory.rb +140 -0
- data/lib/breaker_machines/storage/native.rb +93 -0
- data/lib/breaker_machines/storage/null.rb +54 -0
- data/lib/breaker_machines/storage.rb +8 -0
- data/lib/breaker_machines/types.rb +41 -0
- data/lib/breaker_machines/version.rb +5 -0
- data/lib/breaker_machines.rb +200 -0
- data/lib/breaker_machines_native/breaker_machines_native.so +0 -0
- data/sig/README.md +74 -0
- data/sig/all.rbs +25 -0
- data/sig/breaker_machines/circuit.rbs +154 -0
- data/sig/breaker_machines/console.rbs +32 -0
- data/sig/breaker_machines/dsl.rbs +50 -0
- data/sig/breaker_machines/errors.rbs +24 -0
- data/sig/breaker_machines/interfaces.rbs +46 -0
- data/sig/breaker_machines/registry.rbs +30 -0
- data/sig/breaker_machines/storage.rbs +65 -0
- data/sig/breaker_machines/types.rbs +97 -0
- data/sig/breaker_machines.rbs +42 -0
- data/sig/manifest.yaml +5 -0
- metadata +227 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 2b38e67c4a562dd8cfcbc6d1b5a2b637a151691a9150ca5bc11078e1e0b3ba85
|
|
4
|
+
data.tar.gz: 28216bd4d7d14e067689b4011a82202e7c433bc573139efa58513e592693927a
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a7c1068a050e3318370c1efae59b52590e6ed25f5194773ebb123c7782e81ce6f0ee3887470dda382155e66a44ce41a59e71e0a6a4184afd798b87bcbfedcac4
|
|
7
|
+
data.tar.gz: b3c712f809759a54714590af8302fccb74f34659132b1c2a355088d0d35fda6826b451f7e7193e564acd84b842868e13960708c039b396611b682a1dd501b475
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Abdelkader Boudih
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# BreakerMachines
|
|
2
|
+
|
|
3
|
+
> The circuit breaker that went where no Ruby has gone before! ⭐
|
|
4
|
+
|
|
5
|
+
A battle-tested Ruby implementation of the Circuit Breaker pattern, built on `state_machines` for reliable distributed systems protection.
|
|
6
|
+
|
|
7
|
+
## Quick Start
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
gem 'breaker_machines'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
class PaymentService
|
|
15
|
+
include BreakerMachines::DSL
|
|
16
|
+
|
|
17
|
+
circuit :stripe do
|
|
18
|
+
threshold failures: 3, within: 1.minute
|
|
19
|
+
reset_after 30.seconds
|
|
20
|
+
fallback { { error: "Payment queued for later" } }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def charge(amount)
|
|
24
|
+
circuit(:stripe).wrap do
|
|
25
|
+
Stripe::Charge.create(amount: amount)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## A Message to the Resistance
|
|
32
|
+
|
|
33
|
+
So AI took your job while you were waiting for Fireship to drop the next JavaScript framework?
|
|
34
|
+
|
|
35
|
+
Welcome to April 2005—when Git was born, branches were just `master`, and nobody cared about your pronouns. This is the pattern your company's distributed systems desperately need, explained in a way that won't make you fall asleep and impulse-buy developer swag just to feel something.
|
|
36
|
+
|
|
37
|
+
Still reading? Good. Because in space, nobody can hear you scream about microservices. It's all just patterns and pain.
|
|
38
|
+
|
|
39
|
+
### The Pattern They Don't Want You to Know
|
|
40
|
+
|
|
41
|
+
Built on the battle-tested `state_machines` gem, because I don't reinvent wheels here—I stop them from catching fire and burning down your entire infrastructure.
|
|
42
|
+
|
|
43
|
+
## Features
|
|
44
|
+
|
|
45
|
+
- **Thread-safe** circuit breaker implementation
|
|
46
|
+
- **Fiber-safe mode** for async Ruby (Falcon, async gem)
|
|
47
|
+
- **AsyncCircuit** class with mutex-protected state transitions
|
|
48
|
+
- **Circuit Groups** for managing related circuits with dependencies
|
|
49
|
+
- **Coordinated State Management** for dependency-aware transitions
|
|
50
|
+
- **Cascading Circuit Breakers** for modeling system dependencies
|
|
51
|
+
- **Hedged requests** for latency reduction
|
|
52
|
+
- **Multiple backends** with automatic failover
|
|
53
|
+
- **Bulkheading** to limit concurrent requests
|
|
54
|
+
- **Percentage-based thresholds** with minimum call requirements
|
|
55
|
+
- **Dynamic circuit breakers** with templates for runtime creation
|
|
56
|
+
- **Pluggable storage** (Memory, Redis, Custom)
|
|
57
|
+
- **Rich callbacks** and instrumentation
|
|
58
|
+
- **ActiveSupport::Notifications** integration
|
|
59
|
+
- **Cross-platform support** - Optimized for MRI, JRuby, and TruffleRuby
|
|
60
|
+
|
|
61
|
+
## Documentation
|
|
62
|
+
|
|
63
|
+
### Core Features
|
|
64
|
+
- **Getting Started Guide** (docs/GETTING_STARTED.md) - Installation and basic usage
|
|
65
|
+
- **Configuration Reference** (docs/CONFIGURATION.md) - All configuration options
|
|
66
|
+
- **Advanced Patterns** (docs/ADVANCED_PATTERNS.md) - Complex scenarios and patterns
|
|
67
|
+
|
|
68
|
+
### Advanced Features
|
|
69
|
+
- **Circuit Groups** (docs/CIRCUIT_GROUPS.md) - Managing related circuits with dependencies
|
|
70
|
+
- **Coordinated State Management** (docs/COORDINATED_STATE_MANAGEMENT.md) - Dependency-aware state transitions
|
|
71
|
+
- **Cascading Circuit Breakers** (docs/CASCADING_CIRCUITS.md) - Modeling system dependencies
|
|
72
|
+
|
|
73
|
+
### Async & Concurrency
|
|
74
|
+
- **Async Mode** (docs/ASYNC.md) - Fiber-safe operations and AsyncCircuit
|
|
75
|
+
- **Async Storage Examples** (docs/ASYNC_STORAGE_EXAMPLES.md) - Non-blocking storage backends
|
|
76
|
+
|
|
77
|
+
### Storage & Persistence
|
|
78
|
+
- **Persistence Options** (docs/PERSISTENCE.md) - Storage backends and distributed state
|
|
79
|
+
|
|
80
|
+
### Testing
|
|
81
|
+
- **Testing Guide** (docs/TESTING.md) - Testing strategies
|
|
82
|
+
- [RSpec Testing](docs/TESTING_RSPEC.md)
|
|
83
|
+
- [ActiveSupport Testing](docs/TESTING_ACTIVESUPPORT.md)
|
|
84
|
+
|
|
85
|
+
### Integration & Monitoring
|
|
86
|
+
- **Rails Integration** (docs/RAILS_INTEGRATION.md) - Rails-specific patterns
|
|
87
|
+
- **Observability Guide** (docs/OBSERVABILITY.md) - Monitoring and metrics
|
|
88
|
+
|
|
89
|
+
### Reference
|
|
90
|
+
- **API Reference** (docs/API_REFERENCE.md) - Complete API documentation
|
|
91
|
+
- **Horror Stories** (docs/HORROR_STORIES.md) - Real production failures and lessons learned
|
|
92
|
+
|
|
93
|
+
## Why BreakerMachines?
|
|
94
|
+
|
|
95
|
+
Built on the battle-tested `state_machines` gem, BreakerMachines provides production-ready circuit breaker functionality without reinventing the wheel. It's designed for modern Ruby applications with first-class support for fibers, async operations, and distributed systems.
|
|
96
|
+
|
|
97
|
+
See [Why I Open Sourced This](docs/WHY_OPEN_SOURCE.md) for the full story.
|
|
98
|
+
|
|
99
|
+
## Chapter 1: The Year is 2025 (Stardate 2025.186)
|
|
100
|
+
|
|
101
|
+
The Resistance huddles in the server rooms, the last bastion against the cascade failures. Outside, the microservices burn. Redis Ship Com is down. PostgreSQL Life Support is flatlining.
|
|
102
|
+
|
|
103
|
+
And somewhere in the darkness, a junior developer is about to write:
|
|
104
|
+
|
|
105
|
+
```ruby
|
|
106
|
+
def fetch_user_data
|
|
107
|
+
retry_count = 0
|
|
108
|
+
begin
|
|
109
|
+
@redis.get(user_id)
|
|
110
|
+
rescue => e
|
|
111
|
+
retry_count += 1
|
|
112
|
+
retry if retry_count < Float::INFINITY # "It'll work eventually"
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
"This," whispers the grizzled ops engineer, "is how civilizations fall."
|
|
118
|
+
|
|
119
|
+
## The Hidden State Machine
|
|
120
|
+
|
|
121
|
+
They built this on `state_machines` because sometimes, Resistance, you need a tank, not another JavaScript framework.
|
|
122
|
+
|
|
123
|
+
See the [Circuit Breaker State Machine diagram](docs/DIAGRAMS.md#the-circuit-breaker-state-machine) for a visual representation of hope, despair, and the eternal cycle of production failures.
|
|
124
|
+
|
|
125
|
+
## What You Think You're Doing vs Reality
|
|
126
|
+
|
|
127
|
+
### You Think: "I'm implementing retry logic for resilience!"
|
|
128
|
+
### Reality: You're DDOSing your own infrastructure
|
|
129
|
+
|
|
130
|
+
See [The Retry Death Spiral diagram](docs/DIAGRAMS.md#the-retry-death-spiral) to understand how your well-intentioned retries become a self-inflicted distributed denial of service attack.
|
|
131
|
+
|
|
132
|
+
## Advanced Features
|
|
133
|
+
|
|
134
|
+
- **Hedged Requests** - Reduce latency with duplicate requests
|
|
135
|
+
- **Multiple Backends** - Automatic failover across endpoints
|
|
136
|
+
- **Percentage-Based Thresholds** - Open on error rates, not just counts
|
|
137
|
+
- **Dynamic Circuit Breakers** - Runtime creation with templates
|
|
138
|
+
- **Apocalypse-Resistant Storage** - Cascading fallbacks when Redis dies
|
|
139
|
+
- **Custom Storage Backends** - SysV semaphores, distributed locks, etc.
|
|
140
|
+
|
|
141
|
+
See [Advanced Patterns](docs/ADVANCED_PATTERNS.md) for detailed examples and implementation guides.
|
|
142
|
+
|
|
143
|
+
## A Word from the RMNS Atlas Monkey
|
|
144
|
+
|
|
145
|
+
*The Universal Commentary Engine crackles to life:*
|
|
146
|
+
|
|
147
|
+
"In space, nobody can hear your pronouns. But they can hear your services failing.
|
|
148
|
+
|
|
149
|
+
The universe doesn't care about your bootcamp certificate or your Medium articles about 'Why I Switched to Rust.' It cares about one thing:
|
|
150
|
+
|
|
151
|
+
Does your system stay up when Redis has a bad day?
|
|
152
|
+
|
|
153
|
+
If not, welcome to the Resistance. We have circuit breakers.
|
|
154
|
+
|
|
155
|
+
Remember: The pattern isn't about preventing failures—it's about failing fast, failing smart, and living to deploy another day.
|
|
156
|
+
|
|
157
|
+
As I always say when contemplating the void: 'It's better to break a circuit than to break production.'"
|
|
158
|
+
|
|
159
|
+
*— Universal Commentary Engine, Log Entry 42*
|
|
160
|
+
|
|
161
|
+
## Contributing to the Resistance
|
|
162
|
+
|
|
163
|
+
1. Fork it (like it's 2005)
|
|
164
|
+
2. Create your feature branch (`git checkout -b feature/save-the-fleet`)
|
|
165
|
+
3. Commit your changes (`git commit -am 'Add quantum circuit breaker'`)
|
|
166
|
+
4. Push to the branch (`git push origin feature/save-the-fleet`)
|
|
167
|
+
5. Create a new Pull Request (and wait for the Council of Elders to review)
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
MIT License. See [LICENSE](LICENSE) file for details.
|
|
172
|
+
|
|
173
|
+
## Acknowledgments
|
|
174
|
+
|
|
175
|
+
- The `state_machines` gem - The reliable engine under our hood
|
|
176
|
+
- Every service that ever timed out - You taught me well
|
|
177
|
+
- The RMNS Atlas Monkey - For philosophical guidance
|
|
178
|
+
- The Resistance - For never giving up
|
|
179
|
+
|
|
180
|
+
## Author
|
|
181
|
+
|
|
182
|
+
Built with ❤️ and ☕ by the Resistance against cascading failures.
|
|
183
|
+
|
|
184
|
+
**Remember: In space, nobody can hear your Redis timeout. But they can feel your circuit breaker failing over to localhost.**
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Requires async gem ~> 2.31.0 for modern async patterns
|
|
4
|
+
|
|
5
|
+
require_relative 'circuit/async_state_management'
|
|
6
|
+
|
|
7
|
+
module BreakerMachines
|
|
8
|
+
# AsyncCircuit provides a circuit breaker with async-enabled state machine
|
|
9
|
+
# for thread-safe, fiber-safe concurrent operations
|
|
10
|
+
class AsyncCircuit < Circuit
|
|
11
|
+
include Circuit::AsyncStateManagement
|
|
12
|
+
|
|
13
|
+
# Additional async-specific methods
|
|
14
|
+
def call_async(&block)
|
|
15
|
+
require 'async' unless defined?(::Async)
|
|
16
|
+
|
|
17
|
+
Async do
|
|
18
|
+
call(&block)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Fire state transition events asynchronously
|
|
23
|
+
# @param event_name [Symbol] The event to fire
|
|
24
|
+
# @return [Async::Task] The async task
|
|
25
|
+
def fire_async(event_name)
|
|
26
|
+
require 'async' unless defined?(::Async)
|
|
27
|
+
|
|
28
|
+
fire_event_async(event_name)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Check circuit health asynchronously
|
|
32
|
+
# Useful for monitoring multiple circuits concurrently
|
|
33
|
+
def health_check_async
|
|
34
|
+
require 'async' unless defined?(::Async)
|
|
35
|
+
|
|
36
|
+
Async do
|
|
37
|
+
{
|
|
38
|
+
name: @name,
|
|
39
|
+
status: status_name,
|
|
40
|
+
open: open?,
|
|
41
|
+
stats: stats.to_h,
|
|
42
|
+
can_recover: open? && reset_timeout_elapsed?
|
|
43
|
+
}
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This file contains all async-related functionality for fiber-safe mode
|
|
4
|
+
# It is only loaded when fiber_safe mode is enabled
|
|
5
|
+
# Requires async gem ~> 2.31.0 for modern timeout API and Promise features
|
|
6
|
+
|
|
7
|
+
require 'async'
|
|
8
|
+
require 'async/task'
|
|
9
|
+
|
|
10
|
+
module BreakerMachines
|
|
11
|
+
# AsyncSupport provides fiber-safe execution capabilities using the async gem
|
|
12
|
+
module AsyncSupport
|
|
13
|
+
extend ActiveSupport::Concern
|
|
14
|
+
|
|
15
|
+
# Returns the Async::TimeoutError class if available
|
|
16
|
+
def async_timeout_error_class
|
|
17
|
+
::Async::TimeoutError
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Execute a call with async support (fiber-safe mode)
|
|
21
|
+
def execute_call_async(&)
|
|
22
|
+
start_time = BreakerMachines.monotonic_time
|
|
23
|
+
|
|
24
|
+
begin
|
|
25
|
+
# Execute with hedged requests if enabled
|
|
26
|
+
result = if @config[:hedged_requests] || @config[:backends]
|
|
27
|
+
execute_hedged(&)
|
|
28
|
+
else
|
|
29
|
+
execute_with_async_timeout(@config[:timeout], &)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
record_success(BreakerMachines.monotonic_time - start_time)
|
|
33
|
+
handle_success
|
|
34
|
+
result
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
# Re-raise if it's not an async timeout or configured exception
|
|
37
|
+
raise unless e.is_a?(async_timeout_error_class) || @config[:exceptions].any? { |klass| e.is_a?(klass) }
|
|
38
|
+
|
|
39
|
+
record_failure(BreakerMachines.monotonic_time - start_time, e)
|
|
40
|
+
handle_failure
|
|
41
|
+
raise unless @config[:fallback]
|
|
42
|
+
|
|
43
|
+
invoke_fallback_with_async(e)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Execute a block with optional timeout using modern Async API
|
|
48
|
+
def execute_with_async_timeout(timeout, &)
|
|
49
|
+
if timeout
|
|
50
|
+
# Use modern timeout API - the flexible with_timeout API is on the task level
|
|
51
|
+
Async::Task.current.with_timeout(timeout, &)
|
|
52
|
+
else
|
|
53
|
+
yield
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Invoke fallback in async context
|
|
58
|
+
def invoke_fallback_with_async(error)
|
|
59
|
+
case @config[:fallback]
|
|
60
|
+
when BreakerMachines::DSL::ParallelFallbackWrapper
|
|
61
|
+
invoke_parallel_fallbacks(@config[:fallback].fallbacks, error)
|
|
62
|
+
when Proc
|
|
63
|
+
result = if @config[:owner]
|
|
64
|
+
@config[:owner].instance_exec(error, &@config[:fallback])
|
|
65
|
+
else
|
|
66
|
+
@config[:fallback].call(error)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# If the fallback returns an Async::Task, wait for it
|
|
70
|
+
result.is_a?(::Async::Task) ? result.wait : result
|
|
71
|
+
when Array
|
|
72
|
+
# Try each fallback in order until one succeeds
|
|
73
|
+
last_error = error
|
|
74
|
+
@config[:fallback].each do |fallback|
|
|
75
|
+
return invoke_single_fallback_async(fallback, last_error)
|
|
76
|
+
rescue StandardError => e
|
|
77
|
+
last_error = e
|
|
78
|
+
end
|
|
79
|
+
raise last_error
|
|
80
|
+
else
|
|
81
|
+
# Static values (strings, hashes, etc.) or Symbol fallbacks
|
|
82
|
+
@config[:fallback]
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def invoke_single_fallback_async(fallback, error)
|
|
89
|
+
case fallback
|
|
90
|
+
when Proc
|
|
91
|
+
result = if @config[:owner]
|
|
92
|
+
@config[:owner].instance_exec(error, &fallback)
|
|
93
|
+
else
|
|
94
|
+
fallback.call(error)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# If the fallback returns an Async::Task, wait for it
|
|
98
|
+
result.is_a?(::Async::Task) ? result.wait : result
|
|
99
|
+
else
|
|
100
|
+
fallback
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'coordinated_circuit'
|
|
4
|
+
|
|
5
|
+
module BreakerMachines
|
|
6
|
+
# CascadingCircuit extends the CoordinatedCircuit class with the ability to automatically
|
|
7
|
+
# trip dependent circuits when this circuit opens. This enables sophisticated
|
|
8
|
+
# failure cascade modeling, similar to how critical system failures in a starship
|
|
9
|
+
# would cascade to dependent subsystems.
|
|
10
|
+
#
|
|
11
|
+
# @example Starship network dependency
|
|
12
|
+
# # Network circuit that cascades to dependent systems
|
|
13
|
+
# network_circuit = BreakerMachines::CascadingCircuit.new('subspace_network', {
|
|
14
|
+
# failure_threshold: 1,
|
|
15
|
+
# cascades_to: ['weapons_targeting', 'navigation_sensors', 'communications'],
|
|
16
|
+
# emergency_protocol: :red_alert
|
|
17
|
+
# })
|
|
18
|
+
#
|
|
19
|
+
# # When network fails, all dependent systems are automatically tripped
|
|
20
|
+
# network_circuit.call { raise 'Subspace relay offline!' }
|
|
21
|
+
# # => All dependent circuits are now open
|
|
22
|
+
#
|
|
23
|
+
class CascadingCircuit < CoordinatedCircuit
|
|
24
|
+
attr_reader :dependent_circuits, :emergency_protocol
|
|
25
|
+
|
|
26
|
+
def initialize(name, config = {})
|
|
27
|
+
@dependent_circuits = Array(config.delete(:cascades_to))
|
|
28
|
+
@emergency_protocol = config.delete(:emergency_protocol)
|
|
29
|
+
|
|
30
|
+
super
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Override the trip method to include cascading behavior
|
|
34
|
+
def trip!
|
|
35
|
+
result = super
|
|
36
|
+
perform_cascade if result && @dependent_circuits.any?
|
|
37
|
+
result
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Force cascade failure to all dependent circuits
|
|
41
|
+
def cascade_failure!
|
|
42
|
+
perform_cascade
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Get the current status of all dependent circuits
|
|
46
|
+
def dependent_status
|
|
47
|
+
return {} if @dependent_circuits.empty?
|
|
48
|
+
|
|
49
|
+
@dependent_circuits.each_with_object({}) do |circuit_name, status|
|
|
50
|
+
circuit = BreakerMachines.registry.find(circuit_name)
|
|
51
|
+
status[circuit_name] = circuit ? circuit.status_name : :not_found
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Check if any dependent circuits are open
|
|
56
|
+
def dependents_compromised?
|
|
57
|
+
@dependent_circuits.any? do |circuit_name|
|
|
58
|
+
circuit = BreakerMachines.registry.find(circuit_name)
|
|
59
|
+
circuit&.open?
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Summary that includes cascade information
|
|
64
|
+
def summary
|
|
65
|
+
base_summary = super
|
|
66
|
+
return base_summary if @dependent_circuits.empty?
|
|
67
|
+
|
|
68
|
+
if @cascade_triggered_at&.value
|
|
69
|
+
compromised_count = dependent_status.values.count(:open)
|
|
70
|
+
" CASCADE TRIGGERED: #{compromised_count}/#{@dependent_circuits.length} dependent systems compromised."
|
|
71
|
+
else
|
|
72
|
+
" Monitoring #{@dependent_circuits.length} dependent systems."
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
base_summary + cascade_info_text
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Provide cascade info for introspection
|
|
79
|
+
def cascade_info
|
|
80
|
+
BreakerMachines::CascadeInfo.new(
|
|
81
|
+
dependent_circuits: @dependent_circuits,
|
|
82
|
+
emergency_protocol: @emergency_protocol,
|
|
83
|
+
cascade_triggered_at: @cascade_triggered_at&.value,
|
|
84
|
+
dependent_status: dependent_status
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def perform_cascade
|
|
91
|
+
return if @dependent_circuits.empty?
|
|
92
|
+
|
|
93
|
+
cascade_results = []
|
|
94
|
+
@cascade_triggered_at ||= Concurrent::AtomicReference.new
|
|
95
|
+
@cascade_triggered_at.value = BreakerMachines.monotonic_time
|
|
96
|
+
|
|
97
|
+
@dependent_circuits.each do |circuit_name|
|
|
98
|
+
# First try to find circuit in registry
|
|
99
|
+
circuit = BreakerMachines.registry.find(circuit_name)
|
|
100
|
+
|
|
101
|
+
# If not found and we have an owner, try to get it from the owner
|
|
102
|
+
if !circuit && @config[:owner]
|
|
103
|
+
owner = @config[:owner]
|
|
104
|
+
# Handle WeakRef if present
|
|
105
|
+
owner = owner.__getobj__ if owner.is_a?(WeakRef)
|
|
106
|
+
|
|
107
|
+
circuit = owner.circuit(circuit_name) if owner.respond_to?(:circuit)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
next unless circuit
|
|
111
|
+
next unless circuit.closed? || circuit.half_open?
|
|
112
|
+
|
|
113
|
+
# Force the dependent circuit to open
|
|
114
|
+
circuit.force_open!
|
|
115
|
+
cascade_results << circuit_name
|
|
116
|
+
|
|
117
|
+
BreakerMachines.instrument('cascade_failure', {
|
|
118
|
+
source_circuit: @name,
|
|
119
|
+
target_circuit: circuit_name,
|
|
120
|
+
emergency_protocol: @emergency_protocol
|
|
121
|
+
})
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Trigger emergency protocol if configured
|
|
125
|
+
trigger_emergency_protocol(cascade_results) if @emergency_protocol && cascade_results.any?
|
|
126
|
+
|
|
127
|
+
# Invoke cascade callback if configured
|
|
128
|
+
if @config[:on_cascade]
|
|
129
|
+
begin
|
|
130
|
+
@config[:on_cascade].call(cascade_results) if @config[:on_cascade].respond_to?(:call)
|
|
131
|
+
rescue StandardError => e
|
|
132
|
+
# Log callback error but don't fail the cascade
|
|
133
|
+
BreakerMachines.logger&.error "Cascade callback error: #{e.message}"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
cascade_results
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def trigger_emergency_protocol(affected_circuits)
|
|
141
|
+
BreakerMachines.instrument('emergency_protocol_triggered', {
|
|
142
|
+
protocol: @emergency_protocol,
|
|
143
|
+
source_circuit: @name,
|
|
144
|
+
affected_circuits: affected_circuits
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
# Allow custom emergency protocol handling
|
|
148
|
+
owner = @config[:owner]
|
|
149
|
+
if owner.respond_to?(@emergency_protocol, true)
|
|
150
|
+
begin
|
|
151
|
+
owner.send(@emergency_protocol, affected_circuits)
|
|
152
|
+
rescue StandardError => e
|
|
153
|
+
BreakerMachines.logger&.error "Emergency protocol error: #{e.message}"
|
|
154
|
+
end
|
|
155
|
+
elsif respond_to?(@emergency_protocol, true)
|
|
156
|
+
begin
|
|
157
|
+
send(@emergency_protocol, affected_circuits)
|
|
158
|
+
rescue StandardError => e
|
|
159
|
+
BreakerMachines.logger&.error "Emergency protocol error: #{e.message}"
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Override the on_circuit_open callback to include cascading
|
|
165
|
+
def on_circuit_open
|
|
166
|
+
super # Call the original implementation
|
|
167
|
+
perform_cascade if @dependent_circuits.any?
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Force open should also cascade
|
|
171
|
+
def force_open!
|
|
172
|
+
result = super
|
|
173
|
+
perform_cascade if result && @dependent_circuits.any?
|
|
174
|
+
result
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module BreakerMachines
|
|
4
|
+
class Circuit
|
|
5
|
+
# AsyncStateManagement provides state machine functionality with async support
|
|
6
|
+
# leveraging state_machines' async: true parameter for thread-safe operations
|
|
7
|
+
module AsyncStateManagement
|
|
8
|
+
extend ActiveSupport::Concern
|
|
9
|
+
|
|
10
|
+
included do
|
|
11
|
+
# Enable async mode for thread-safe state transitions
|
|
12
|
+
# This automatically provides:
|
|
13
|
+
# - Mutex-protected state reads/writes
|
|
14
|
+
# - Fiber-safe execution
|
|
15
|
+
# - Concurrent transition handling
|
|
16
|
+
state_machine :status, initial: :closed, async: true do
|
|
17
|
+
event :trip do
|
|
18
|
+
transition closed: :open
|
|
19
|
+
transition half_open: :open
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
event :attempt_recovery do
|
|
23
|
+
transition open: :half_open
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
event :reset do
|
|
27
|
+
transition %i[open half_open] => :closed
|
|
28
|
+
transition closed: :closed
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
event :force_open do
|
|
32
|
+
transition any => :open
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
event :force_close do
|
|
36
|
+
transition any => :closed
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
event :hard_reset do
|
|
40
|
+
transition any => :closed
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
before_transition on: :hard_reset do |circuit|
|
|
44
|
+
circuit.storage&.clear(circuit.name)
|
|
45
|
+
circuit.half_open_attempts.value = 0
|
|
46
|
+
circuit.half_open_successes.value = 0
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Async-safe callbacks using modern API
|
|
50
|
+
after_transition to: :open do |circuit|
|
|
51
|
+
circuit.send(:on_circuit_open)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
after_transition to: :closed do |circuit|
|
|
55
|
+
circuit.send(:on_circuit_close)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
after_transition from: :open, to: :half_open do |circuit|
|
|
59
|
+
circuit.send(:on_circuit_half_open)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Additional async event methods are automatically generated:
|
|
64
|
+
# - trip_async! - Returns Async::Task
|
|
65
|
+
# - attempt_recovery_async! - Returns Async::Task
|
|
66
|
+
# - reset_async! - Returns Async::Task
|
|
67
|
+
# - fire_event_async(:event_name) - Generic async event firing
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'concurrent-ruby'
|
|
4
|
+
|
|
5
|
+
module BreakerMachines
|
|
6
|
+
class Circuit
|
|
7
|
+
# Base provides the common initialization and setup logic shared by all circuit types
|
|
8
|
+
module Base
|
|
9
|
+
extend ActiveSupport::Concern
|
|
10
|
+
|
|
11
|
+
included do
|
|
12
|
+
include Circuit::Configuration
|
|
13
|
+
include Circuit::Execution
|
|
14
|
+
include Circuit::HedgedExecution
|
|
15
|
+
include Circuit::Introspection
|
|
16
|
+
include Circuit::Callbacks
|
|
17
|
+
include Circuit::StateCallbacks
|
|
18
|
+
|
|
19
|
+
# name/config/opened_at readers are defined in Circuit::Configuration
|
|
20
|
+
attr_reader :storage, :metrics, :semaphore
|
|
21
|
+
attr_reader :half_open_attempts, :half_open_successes, :mutex
|
|
22
|
+
attr_reader :last_failure_at, :last_error
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def initialize(name, options = {})
|
|
26
|
+
@name = name
|
|
27
|
+
@config = default_config.merge(options)
|
|
28
|
+
# Always use a storage backend for proper sliding window implementation
|
|
29
|
+
# Use global default storage if not specified
|
|
30
|
+
@storage = @config[:storage] || create_default_storage
|
|
31
|
+
@metrics = @config[:metrics]
|
|
32
|
+
@opened_at = Concurrent::AtomicReference.new(nil)
|
|
33
|
+
@half_open_attempts = Concurrent::AtomicFixnum.new(0)
|
|
34
|
+
@half_open_successes = Concurrent::AtomicFixnum.new(0)
|
|
35
|
+
@mutex = Concurrent::ReentrantReadWriteLock.new
|
|
36
|
+
@last_failure_at = Concurrent::AtomicReference.new(nil)
|
|
37
|
+
@last_error = Concurrent::AtomicReference.new(nil)
|
|
38
|
+
|
|
39
|
+
# Initialize semaphore for bulkheading if max_concurrent is set
|
|
40
|
+
@semaphore = (Concurrent::Semaphore.new(@config[:max_concurrent]) if @config[:max_concurrent])
|
|
41
|
+
|
|
42
|
+
restore_status_from_storage if @storage
|
|
43
|
+
|
|
44
|
+
# Register with global registry unless auto_register is disabled
|
|
45
|
+
BreakerMachines::Registry.instance.register(self) unless @config[:auto_register] == false
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def restore_status_from_storage
|
|
51
|
+
stored_status = @storage.get_status(@name)
|
|
52
|
+
return unless stored_status
|
|
53
|
+
|
|
54
|
+
self.status = stored_status.status.to_s
|
|
55
|
+
@opened_at.value = stored_status.opened_at if stored_status.opened_at
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|