breaker_machines 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +25 -3
- data/lib/breaker_machines/async_circuit.rb +47 -0
- data/lib/breaker_machines/async_support.rb +4 -3
- data/lib/breaker_machines/cascading_circuit.rb +5 -3
- data/lib/breaker_machines/circuit/async_state_management.rb +71 -0
- data/lib/breaker_machines/circuit/base.rb +58 -0
- data/lib/breaker_machines/circuit/callbacks.rb +7 -12
- data/lib/breaker_machines/circuit/configuration.rb +6 -26
- data/lib/breaker_machines/circuit/coordinated_state_management.rb +117 -0
- data/lib/breaker_machines/circuit/hedged_execution.rb +115 -0
- data/lib/breaker_machines/circuit/introspection.rb +1 -0
- data/lib/breaker_machines/circuit/state_callbacks.rb +60 -0
- data/lib/breaker_machines/circuit/state_management.rb +14 -61
- data/lib/breaker_machines/circuit.rb +1 -7
- data/lib/breaker_machines/circuit_group.rb +153 -0
- data/lib/breaker_machines/coordinated_circuit.rb +10 -0
- data/lib/breaker_machines/dsl.rb +2 -2
- data/lib/breaker_machines/errors.rb +20 -0
- data/lib/breaker_machines/hedged_async_support.rb +29 -36
- data/lib/breaker_machines/version.rb +1 -1
- data/lib/breaker_machines.rb +17 -0
- metadata +11 -4
- data/lib/breaker_machines/hedged_execution.rb +0 -113
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 97bc515f00b4402af03e813a121969745726a3635a8475bd3066877294f535a0
|
4
|
+
data.tar.gz: 20370133d03443108818133a98790194a4669a5b9daafdb1808d537b3915bb32
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 365d62466c1dae963c4f26e4f7920868358d9a39cd627bbc017ee7f8d36807e03ae808d1d4057dfee54b2edfc3e1dadaf36bf687b33aed8201bafac1a38f168f
|
7
|
+
data.tar.gz: 54fe1284d6710fab9a347b05821c41ea98f04212b224752d6bfc4f7d3fd995febb431acfee57f83a30859b2a7c7a138da59bb1c16ebf15032f018530c72fe397
|
data/README.md
CHANGED
@@ -44,6 +44,10 @@ Built on the battle-tested `state_machines` gem, because I don't reinvent wheels
|
|
44
44
|
|
45
45
|
- **Thread-safe** circuit breaker implementation
|
46
46
|
- **Fiber-safe mode** for async Ruby (Falcon, async gem)
|
47
|
+
- **AsyncCircuit** class with mutex-protected state transitions
|
48
|
+
- **Circuit Groups** for managing related circuits with dependencies
|
49
|
+
- **Coordinated State Management** for dependency-aware transitions
|
50
|
+
- **Cascading Circuit Breakers** for modeling system dependencies
|
47
51
|
- **Hedged requests** for latency reduction
|
48
52
|
- **Multiple backends** with automatic failover
|
49
53
|
- **Bulkheading** to limit concurrent requests
|
@@ -52,21 +56,39 @@ Built on the battle-tested `state_machines` gem, because I don't reinvent wheels
|
|
52
56
|
- **Pluggable storage** (Memory, Redis, Custom)
|
53
57
|
- **Rich callbacks** and instrumentation
|
54
58
|
- **ActiveSupport::Notifications** integration
|
59
|
+
- **Cross-platform support** - Optimized for MRI, JRuby, and TruffleRuby
|
55
60
|
|
56
61
|
## Documentation
|
57
62
|
|
63
|
+
### Core Features
|
58
64
|
- **Getting Started Guide** (docs/GETTING_STARTED.md) - Installation and basic usage
|
59
65
|
- **Configuration Reference** (docs/CONFIGURATION.md) - All configuration options
|
60
66
|
- **Advanced Patterns** (docs/ADVANCED_PATTERNS.md) - Complex scenarios and patterns
|
67
|
+
|
68
|
+
### Advanced Features
|
69
|
+
- **Circuit Groups** (docs/CIRCUIT_GROUPS.md) - Managing related circuits with dependencies
|
70
|
+
- **Coordinated State Management** (docs/COORDINATED_STATE_MANAGEMENT.md) - Dependency-aware state transitions
|
71
|
+
- **Cascading Circuit Breakers** (docs/CASCADING_CIRCUITS.md) - Modeling system dependencies
|
72
|
+
|
73
|
+
### Async & Concurrency
|
74
|
+
- **Async Mode** (docs/ASYNC.md) - Fiber-safe operations and AsyncCircuit
|
75
|
+
- **Async Storage Examples** (docs/ASYNC_STORAGE_EXAMPLES.md) - Non-blocking storage backends
|
76
|
+
|
77
|
+
### Storage & Persistence
|
61
78
|
- **Persistence Options** (docs/PERSISTENCE.md) - Storage backends and distributed state
|
62
|
-
|
63
|
-
|
79
|
+
|
80
|
+
### Testing
|
64
81
|
- **Testing Guide** (docs/TESTING.md) - Testing strategies
|
65
82
|
- [RSpec Testing](docs/TESTING_RSPEC.md)
|
66
83
|
- [ActiveSupport Testing](docs/TESTING_ACTIVESUPPORT.md)
|
84
|
+
|
85
|
+
### Integration & Monitoring
|
67
86
|
- **Rails Integration** (docs/RAILS_INTEGRATION.md) - Rails-specific patterns
|
68
|
-
- **
|
87
|
+
- **Observability Guide** (docs/OBSERVABILITY.md) - Monitoring and metrics
|
88
|
+
|
89
|
+
### Reference
|
69
90
|
- **API Reference** (docs/API_REFERENCE.md) - Complete API documentation
|
91
|
+
- **Horror Stories** (docs/HORROR_STORIES.md) - Real production failures and lessons learned
|
70
92
|
|
71
93
|
## Why BreakerMachines?
|
72
94
|
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Requires async gem ~> 2.31.0 for modern async patterns
|
4
|
+
|
5
|
+
require_relative 'circuit/async_state_management'
|
6
|
+
|
7
|
+
module BreakerMachines
|
8
|
+
# AsyncCircuit provides a circuit breaker with async-enabled state machine
|
9
|
+
# for thread-safe, fiber-safe concurrent operations
|
10
|
+
class AsyncCircuit < Circuit
|
11
|
+
include Circuit::AsyncStateManagement
|
12
|
+
|
13
|
+
# Additional async-specific methods
|
14
|
+
def call_async(&block)
|
15
|
+
require 'async' unless defined?(::Async)
|
16
|
+
|
17
|
+
Async do
|
18
|
+
call(&block)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# Fire state transition events asynchronously
|
23
|
+
# @param event_name [Symbol] The event to fire
|
24
|
+
# @return [Async::Task] The async task
|
25
|
+
def fire_async(event_name)
|
26
|
+
require 'async' unless defined?(::Async)
|
27
|
+
|
28
|
+
fire_event_async(event_name)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Check circuit health asynchronously
|
32
|
+
# Useful for monitoring multiple circuits concurrently
|
33
|
+
def health_check_async
|
34
|
+
require 'async' unless defined?(::Async)
|
35
|
+
|
36
|
+
Async do
|
37
|
+
{
|
38
|
+
name: @name,
|
39
|
+
status: status_name,
|
40
|
+
open: open?,
|
41
|
+
stats: stats.to_h,
|
42
|
+
can_recover: open? && reset_timeout_elapsed?
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
# This file contains all async-related functionality for fiber-safe mode
|
4
4
|
# It is only loaded when fiber_safe mode is enabled
|
5
|
+
# Requires async gem ~> 2.31.0 for modern timeout API and Promise features
|
5
6
|
|
6
7
|
require 'async'
|
7
8
|
require 'async/task'
|
@@ -43,11 +44,11 @@ module BreakerMachines
|
|
43
44
|
end
|
44
45
|
end
|
45
46
|
|
46
|
-
# Execute a block with optional timeout using Async
|
47
|
+
# Execute a block with optional timeout using modern Async API
|
47
48
|
def execute_with_async_timeout(timeout, &)
|
48
49
|
if timeout
|
49
|
-
# Use
|
50
|
-
|
50
|
+
# Use modern timeout API - the flexible with_timeout API is on the task level
|
51
|
+
Async::Task.current.with_timeout(timeout, &)
|
51
52
|
else
|
52
53
|
yield
|
53
54
|
end
|
@@ -1,7 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative 'coordinated_circuit'
|
4
|
+
|
3
5
|
module BreakerMachines
|
4
|
-
# CascadingCircuit extends the
|
6
|
+
# CascadingCircuit extends the CoordinatedCircuit class with the ability to automatically
|
5
7
|
# trip dependent circuits when this circuit opens. This enables sophisticated
|
6
8
|
# failure cascade modeling, similar to how critical system failures in a starship
|
7
9
|
# would cascade to dependent subsystems.
|
@@ -18,7 +20,7 @@ module BreakerMachines
|
|
18
20
|
# network_circuit.call { raise 'Subspace relay offline!' }
|
19
21
|
# # => All dependent circuits are now open
|
20
22
|
#
|
21
|
-
class CascadingCircuit <
|
23
|
+
class CascadingCircuit < CoordinatedCircuit
|
22
24
|
attr_reader :dependent_circuits, :emergency_protocol
|
23
25
|
|
24
26
|
def initialize(name, config = {})
|
@@ -144,7 +146,7 @@ module BreakerMachines
|
|
144
146
|
|
145
147
|
# Allow custom emergency protocol handling
|
146
148
|
owner = @config[:owner]
|
147
|
-
if owner
|
149
|
+
if owner.respond_to?(@emergency_protocol, true)
|
148
150
|
begin
|
149
151
|
owner.send(@emergency_protocol, affected_circuits)
|
150
152
|
rescue StandardError => e
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module BreakerMachines
|
4
|
+
class Circuit
|
5
|
+
# AsyncStateManagement provides state machine functionality with async support
|
6
|
+
# leveraging state_machines' async: true parameter for thread-safe operations
|
7
|
+
module AsyncStateManagement
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
included do
|
11
|
+
# Enable async mode for thread-safe state transitions
|
12
|
+
# This automatically provides:
|
13
|
+
# - Mutex-protected state reads/writes
|
14
|
+
# - Fiber-safe execution
|
15
|
+
# - Concurrent transition handling
|
16
|
+
state_machine :status, initial: :closed, async: true do
|
17
|
+
event :trip do
|
18
|
+
transition closed: :open
|
19
|
+
transition half_open: :open
|
20
|
+
end
|
21
|
+
|
22
|
+
event :attempt_recovery do
|
23
|
+
transition open: :half_open
|
24
|
+
end
|
25
|
+
|
26
|
+
event :reset do
|
27
|
+
transition %i[open half_open] => :closed
|
28
|
+
transition closed: :closed
|
29
|
+
end
|
30
|
+
|
31
|
+
event :force_open do
|
32
|
+
transition any => :open
|
33
|
+
end
|
34
|
+
|
35
|
+
event :force_close do
|
36
|
+
transition any => :closed
|
37
|
+
end
|
38
|
+
|
39
|
+
event :hard_reset do
|
40
|
+
transition any => :closed
|
41
|
+
end
|
42
|
+
|
43
|
+
before_transition on: :hard_reset do |circuit|
|
44
|
+
circuit.storage.clear(circuit.name) if circuit.storage
|
45
|
+
circuit.half_open_attempts.value = 0
|
46
|
+
circuit.half_open_successes.value = 0
|
47
|
+
end
|
48
|
+
|
49
|
+
# Async-safe callbacks using modern API
|
50
|
+
after_transition to: :open do |circuit|
|
51
|
+
circuit.send(:on_circuit_open)
|
52
|
+
end
|
53
|
+
|
54
|
+
after_transition to: :closed do |circuit|
|
55
|
+
circuit.send(:on_circuit_close)
|
56
|
+
end
|
57
|
+
|
58
|
+
after_transition from: :open, to: :half_open do |circuit|
|
59
|
+
circuit.send(:on_circuit_half_open)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Additional async event methods are automatically generated:
|
64
|
+
# - trip_async! - Returns Async::Task
|
65
|
+
# - attempt_recovery_async! - Returns Async::Task
|
66
|
+
# - reset_async! - Returns Async::Task
|
67
|
+
# - fire_event_async(:event_name) - Generic async event firing
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'concurrent-ruby'
|
4
|
+
|
5
|
+
module BreakerMachines
|
6
|
+
class Circuit
|
7
|
+
# Base provides the common initialization and setup logic shared by all circuit types
|
8
|
+
module Base
|
9
|
+
extend ActiveSupport::Concern
|
10
|
+
|
11
|
+
included do
|
12
|
+
include Circuit::Configuration
|
13
|
+
include Circuit::Execution
|
14
|
+
include Circuit::HedgedExecution
|
15
|
+
include Circuit::Introspection
|
16
|
+
include Circuit::Callbacks
|
17
|
+
include Circuit::StateCallbacks
|
18
|
+
|
19
|
+
attr_reader :name, :config, :opened_at, :storage, :metrics, :semaphore
|
20
|
+
attr_reader :half_open_attempts, :half_open_successes, :mutex
|
21
|
+
attr_reader :last_failure_at, :last_error
|
22
|
+
end
|
23
|
+
|
24
|
+
def initialize(name, options = {})
|
25
|
+
@name = name
|
26
|
+
@config = default_config.merge(options)
|
27
|
+
# Always use a storage backend for proper sliding window implementation
|
28
|
+
# Use global default storage if not specified
|
29
|
+
@storage = @config[:storage] || create_default_storage
|
30
|
+
@metrics = @config[:metrics]
|
31
|
+
@opened_at = Concurrent::AtomicReference.new(nil)
|
32
|
+
@half_open_attempts = Concurrent::AtomicFixnum.new(0)
|
33
|
+
@half_open_successes = Concurrent::AtomicFixnum.new(0)
|
34
|
+
@mutex = Concurrent::ReentrantReadWriteLock.new
|
35
|
+
@last_failure_at = Concurrent::AtomicReference.new(nil)
|
36
|
+
@last_error = Concurrent::AtomicReference.new(nil)
|
37
|
+
|
38
|
+
# Initialize semaphore for bulkheading if max_concurrent is set
|
39
|
+
@semaphore = (Concurrent::Semaphore.new(@config[:max_concurrent]) if @config[:max_concurrent])
|
40
|
+
|
41
|
+
restore_status_from_storage if @storage
|
42
|
+
|
43
|
+
# Register with global registry unless auto_register is disabled
|
44
|
+
BreakerMachines::Registry.instance.register(self) unless @config[:auto_register] == false
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def restore_status_from_storage
|
50
|
+
stored_status = @storage.get_status(@name)
|
51
|
+
return unless stored_status
|
52
|
+
|
53
|
+
self.status = stored_status.status.to_s
|
54
|
+
@opened_at.value = stored_status.opened_at if stored_status.opened_at
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -120,19 +120,14 @@ module BreakerMachines
|
|
120
120
|
end
|
121
121
|
end
|
122
122
|
|
123
|
-
|
124
|
-
begin
|
125
|
-
Timeout.timeout(5) do # reasonable timeout for fallbacks
|
126
|
-
loop do
|
127
|
-
return result_queue.pop unless result_queue.empty?
|
128
|
-
|
129
|
-
raise error_queue.pop if error_queue.size >= fallbacks.size
|
123
|
+
threads.each(&:join)
|
130
124
|
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
125
|
+
if result_queue.empty?
|
126
|
+
errors = []
|
127
|
+
errors << error_queue.pop until error_queue.empty?
|
128
|
+
raise BreakerMachines::ParallelFallbackError.new('All parallel fallbacks failed', errors)
|
129
|
+
else
|
130
|
+
result_queue.pop
|
136
131
|
end
|
137
132
|
end
|
138
133
|
end
|
@@ -11,30 +11,6 @@ module BreakerMachines
|
|
11
11
|
attr_reader :name, :config, :opened_at
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(name, options = {})
|
15
|
-
@name = name
|
16
|
-
@config = default_config.merge(options)
|
17
|
-
# Always use a storage backend for proper sliding window implementation
|
18
|
-
# Use global default storage if not specified
|
19
|
-
@storage = @config[:storage] || create_default_storage
|
20
|
-
@metrics = @config[:metrics]
|
21
|
-
@opened_at = Concurrent::AtomicReference.new(nil)
|
22
|
-
@half_open_attempts = Concurrent::AtomicFixnum.new(0)
|
23
|
-
@half_open_successes = Concurrent::AtomicFixnum.new(0)
|
24
|
-
@mutex = Concurrent::ReentrantReadWriteLock.new
|
25
|
-
@last_failure_at = Concurrent::AtomicReference.new(nil)
|
26
|
-
@last_error = Concurrent::AtomicReference.new(nil)
|
27
|
-
|
28
|
-
# Initialize semaphore for bulkheading if max_concurrent is set
|
29
|
-
@semaphore = (Concurrent::Semaphore.new(@config[:max_concurrent]) if @config[:max_concurrent])
|
30
|
-
|
31
|
-
super() # Initialize state machine
|
32
|
-
restore_status_from_storage if @storage
|
33
|
-
|
34
|
-
# Register with global registry unless auto_register is disabled
|
35
|
-
BreakerMachines::Registry.instance.register(self) unless @config[:auto_register] == false
|
36
|
-
end
|
37
|
-
|
38
14
|
private
|
39
15
|
|
40
16
|
def default_config
|
@@ -78,8 +54,12 @@ module BreakerMachines
|
|
78
54
|
when :null
|
79
55
|
BreakerMachines::Storage::Null.new
|
80
56
|
else
|
81
|
-
# Allow for custom storage class names
|
82
|
-
BreakerMachines.config.default_storage.new
|
57
|
+
# Allow for custom storage class names or instances
|
58
|
+
if BreakerMachines.config.default_storage.respond_to?(:new)
|
59
|
+
BreakerMachines.config.default_storage.new
|
60
|
+
else
|
61
|
+
BreakerMachines.config.default_storage
|
62
|
+
end
|
83
63
|
end
|
84
64
|
end
|
85
65
|
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module BreakerMachines
|
4
|
+
class Circuit
|
5
|
+
# CoordinatedStateManagement extends the base state machine with coordinated guards
|
6
|
+
# that allow circuits to make transitions based on the state of other circuits.
|
7
|
+
module CoordinatedStateManagement
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
included do
|
11
|
+
# Override the state machine to add coordinated guards
|
12
|
+
state_machine :status, initial: :closed do
|
13
|
+
event :trip do
|
14
|
+
transition closed: :open
|
15
|
+
transition half_open: :open
|
16
|
+
end
|
17
|
+
|
18
|
+
event :attempt_recovery do
|
19
|
+
transition open: :half_open,
|
20
|
+
if: ->(circuit) { circuit.recovery_allowed? }
|
21
|
+
end
|
22
|
+
|
23
|
+
event :reset do
|
24
|
+
transition %i[open half_open] => :closed,
|
25
|
+
if: ->(circuit) { circuit.reset_allowed? }
|
26
|
+
transition closed: :closed
|
27
|
+
end
|
28
|
+
|
29
|
+
event :force_open do
|
30
|
+
transition any => :open
|
31
|
+
end
|
32
|
+
|
33
|
+
event :force_close do
|
34
|
+
transition any => :closed
|
35
|
+
end
|
36
|
+
|
37
|
+
event :hard_reset do
|
38
|
+
transition any => :closed
|
39
|
+
end
|
40
|
+
|
41
|
+
before_transition on: :hard_reset do |circuit|
|
42
|
+
circuit.storage.clear(circuit.name) if circuit.storage
|
43
|
+
circuit.half_open_attempts.value = 0
|
44
|
+
circuit.half_open_successes.value = 0
|
45
|
+
end
|
46
|
+
|
47
|
+
after_transition to: :open do |circuit|
|
48
|
+
circuit.send(:on_circuit_open)
|
49
|
+
end
|
50
|
+
|
51
|
+
after_transition to: :closed do |circuit|
|
52
|
+
circuit.send(:on_circuit_close)
|
53
|
+
end
|
54
|
+
|
55
|
+
after_transition from: :open, to: :half_open do |circuit|
|
56
|
+
circuit.send(:on_circuit_half_open)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# Check if this circuit can attempt recovery
|
62
|
+
# For cascading circuits, this checks if dependent circuits allow it
|
63
|
+
def recovery_allowed?
|
64
|
+
return true unless respond_to?(:dependent_circuits) && dependent_circuits.any?
|
65
|
+
|
66
|
+
# Don't attempt recovery if any critical dependencies are still down
|
67
|
+
!has_critical_dependencies_down?
|
68
|
+
end
|
69
|
+
|
70
|
+
# Check if this circuit can reset to closed
|
71
|
+
# For cascading circuits, ensures dependencies are healthy
|
72
|
+
def reset_allowed?
|
73
|
+
return true unless respond_to?(:dependent_circuits) && dependent_circuits.any?
|
74
|
+
|
75
|
+
# Only reset if all dependencies are in acceptable states
|
76
|
+
all_dependencies_healthy?
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
|
81
|
+
# Check if any critical dependencies are down
|
82
|
+
def has_critical_dependencies_down?
|
83
|
+
return false unless respond_to?(:dependent_circuits)
|
84
|
+
|
85
|
+
dependent_circuits.any? do |circuit_name|
|
86
|
+
circuit = find_dependent_circuit(circuit_name)
|
87
|
+
circuit&.open?
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
# Check if all dependencies are in healthy states
|
92
|
+
def all_dependencies_healthy?
|
93
|
+
return true unless respond_to?(:dependent_circuits)
|
94
|
+
|
95
|
+
dependent_circuits.all? do |circuit_name|
|
96
|
+
circuit = find_dependent_circuit(circuit_name)
|
97
|
+
circuit.nil? || circuit.closed? || circuit.half_open?
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# Find a dependent circuit by name
|
102
|
+
def find_dependent_circuit(circuit_name)
|
103
|
+
# First try registry
|
104
|
+
circuit = BreakerMachines.registry.find(circuit_name)
|
105
|
+
|
106
|
+
# If not found and we have an owner, try to get it from the owner
|
107
|
+
if !circuit && @config[:owner]
|
108
|
+
owner = @config[:owner]
|
109
|
+
owner = owner.__getobj__ if owner.is_a?(WeakRef)
|
110
|
+
circuit = owner.circuit(circuit_name) if owner.respond_to?(:circuit)
|
111
|
+
end
|
112
|
+
|
113
|
+
circuit
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'concurrent'
|
4
|
+
require 'timeout'
|
5
|
+
|
6
|
+
module BreakerMachines
|
7
|
+
class Circuit
|
8
|
+
# HedgedExecution provides hedged request functionality for circuit breakers
|
9
|
+
# Hedged requests improve latency by sending duplicate requests to multiple backends
|
10
|
+
# and returning the first successful response
|
11
|
+
module HedgedExecution
|
12
|
+
extend ActiveSupport::Concern
|
13
|
+
|
14
|
+
# Execute a hedged request pattern
|
15
|
+
def execute_hedged(&)
|
16
|
+
return execute_single_hedged(&) unless @config[:backends]&.any?
|
17
|
+
|
18
|
+
execute_multi_backend_hedged
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
# Execute hedged request with a single backend (original block)
|
24
|
+
def execute_single_hedged(&block)
|
25
|
+
return yield unless hedged_requests_enabled?
|
26
|
+
|
27
|
+
max_requests = @config[:max_hedged_requests] || 2
|
28
|
+
delay_ms = @config[:hedging_delay] || 50
|
29
|
+
|
30
|
+
if @config[:fiber_safe]
|
31
|
+
execute_hedged_async(Array.new(max_requests) { block }, delay_ms)
|
32
|
+
else
|
33
|
+
execute_hedged_sync(Array.new(max_requests) { block }, delay_ms)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Execute hedged requests across multiple backends
|
38
|
+
def execute_multi_backend_hedged
|
39
|
+
backends = @config[:backends]
|
40
|
+
return backends.first.call if backends.size == 1
|
41
|
+
|
42
|
+
if @config[:fiber_safe]
|
43
|
+
execute_hedged_async(backends, @config[:hedging_delay] || 0)
|
44
|
+
else
|
45
|
+
execute_hedged_sync(backends, @config[:hedging_delay] || 0)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# Synchronous hedged execution using threads
|
50
|
+
def execute_hedged_sync(callables, delay_ms)
|
51
|
+
result_queue = Queue.new
|
52
|
+
error_queue = Queue.new
|
53
|
+
threads = []
|
54
|
+
cancelled = Concurrent::AtomicBoolean.new(false)
|
55
|
+
|
56
|
+
callables.each_with_index do |callable, index|
|
57
|
+
# Add delay for hedge requests (not the first one)
|
58
|
+
sleep(delay_ms / 1000.0) if index.positive? && delay_ms.positive?
|
59
|
+
|
60
|
+
# Skip if already got a result
|
61
|
+
break if cancelled.value
|
62
|
+
|
63
|
+
threads << Thread.new do
|
64
|
+
unless cancelled.value
|
65
|
+
begin
|
66
|
+
result = callable.call
|
67
|
+
result_queue << result unless cancelled.value
|
68
|
+
cancelled.value = true
|
69
|
+
rescue StandardError => e
|
70
|
+
error_queue << e unless cancelled.value
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# Wait for first result or all errors
|
77
|
+
begin
|
78
|
+
Timeout.timeout(@config[:timeout] || 30) do
|
79
|
+
# Check for successful result
|
80
|
+
loop do
|
81
|
+
unless result_queue.empty?
|
82
|
+
result = result_queue.pop
|
83
|
+
cancelled.value = true
|
84
|
+
return result
|
85
|
+
end
|
86
|
+
|
87
|
+
# Check if all requests failed
|
88
|
+
raise error_queue.pop if error_queue.size >= callables.size
|
89
|
+
|
90
|
+
# Small sleep to prevent busy waiting
|
91
|
+
sleep 0.001
|
92
|
+
end
|
93
|
+
end
|
94
|
+
ensure
|
95
|
+
# Cancel remaining threads
|
96
|
+
cancelled.value = true
|
97
|
+
threads.each(&:kill)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# Async hedged execution (requires async support)
|
102
|
+
def execute_hedged_async(callables, delay_ms)
|
103
|
+
# This will be implemented when async support is loaded
|
104
|
+
# For now, fall back to sync implementation
|
105
|
+
return execute_hedged_sync(callables, delay_ms) unless respond_to?(:execute_hedged_with_async)
|
106
|
+
|
107
|
+
execute_hedged_with_async(callables, delay_ms)
|
108
|
+
end
|
109
|
+
|
110
|
+
def hedged_requests_enabled?
|
111
|
+
@config[:hedged_requests] == true
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -6,6 +6,7 @@ module BreakerMachines
|
|
6
6
|
# and generating human-readable summaries of circuit status.
|
7
7
|
module Introspection
|
8
8
|
extend ActiveSupport::Concern
|
9
|
+
|
9
10
|
# State check methods are automatically generated by state_machines:
|
10
11
|
# - open? (returns true when status == :open)
|
11
12
|
# - closed? (returns true when status == :closed)
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module BreakerMachines
|
4
|
+
class Circuit
|
5
|
+
# StateCallbacks provides the common callback methods shared by all state management modules
|
6
|
+
module StateCallbacks
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def on_circuit_open
|
12
|
+
@opened_at.value = BreakerMachines.monotonic_time
|
13
|
+
@storage&.set_status(@name, :open, @opened_at.value)
|
14
|
+
if @storage.respond_to?(:record_event_with_details)
|
15
|
+
@storage.record_event_with_details(@name, :state_change, 0,
|
16
|
+
new_state: :open)
|
17
|
+
end
|
18
|
+
invoke_callback(:on_open)
|
19
|
+
BreakerMachines.instrument('opened', circuit: @name)
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_circuit_close
|
23
|
+
@opened_at.value = nil
|
24
|
+
@last_error.value = nil
|
25
|
+
@last_failure_at.value = nil
|
26
|
+
@storage&.set_status(@name, :closed)
|
27
|
+
if @storage.respond_to?(:record_event_with_details)
|
28
|
+
@storage.record_event_with_details(@name, :state_change, 0,
|
29
|
+
new_state: :closed)
|
30
|
+
end
|
31
|
+
invoke_callback(:on_close)
|
32
|
+
BreakerMachines.instrument('closed', circuit: @name)
|
33
|
+
end
|
34
|
+
|
35
|
+
def on_circuit_half_open
|
36
|
+
@half_open_attempts.value = 0
|
37
|
+
@half_open_successes.value = 0
|
38
|
+
@storage&.set_status(@name, :half_open)
|
39
|
+
if @storage.respond_to?(:record_event_with_details)
|
40
|
+
@storage.record_event_with_details(@name, :state_change, 0,
|
41
|
+
new_state: :half_open)
|
42
|
+
end
|
43
|
+
invoke_callback(:on_half_open)
|
44
|
+
BreakerMachines.instrument('half_opened', circuit: @name)
|
45
|
+
end
|
46
|
+
|
47
|
+
def reset_timeout_elapsed?
|
48
|
+
return false unless @opened_at.value
|
49
|
+
|
50
|
+
# Add jitter to prevent thundering herd
|
51
|
+
jitter_factor = @config[:reset_timeout_jitter] || 0.25
|
52
|
+
# Calculate random jitter between -jitter_factor and +jitter_factor
|
53
|
+
jitter_multiplier = 1.0 + (((rand * 2) - 1) * jitter_factor)
|
54
|
+
timeout_with_jitter = @config[:reset_timeout] * jitter_multiplier
|
55
|
+
|
56
|
+
BreakerMachines.monotonic_time - @opened_at.value >= timeout_with_jitter
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -6,6 +6,7 @@ module BreakerMachines
|
|
6
6
|
# managing transitions between closed, open, and half-open states.
|
7
7
|
module StateManagement
|
8
8
|
extend ActiveSupport::Concern
|
9
|
+
|
9
10
|
included do
|
10
11
|
state_machine :status, initial: :closed do
|
11
12
|
event :trip do
|
@@ -30,77 +31,29 @@ module BreakerMachines
|
|
30
31
|
transition any => :closed
|
31
32
|
end
|
32
33
|
|
33
|
-
|
34
|
+
event :hard_reset do
|
35
|
+
transition any => :closed
|
36
|
+
end
|
37
|
+
|
38
|
+
before_transition on: :hard_reset do |circuit|
|
39
|
+
circuit.storage.clear(circuit.name) if circuit.storage
|
40
|
+
circuit.half_open_attempts.value = 0
|
41
|
+
circuit.half_open_successes.value = 0
|
42
|
+
end
|
43
|
+
|
44
|
+
after_transition to: :open do |circuit|
|
34
45
|
circuit.send(:on_circuit_open)
|
35
46
|
end
|
36
47
|
|
37
|
-
after_transition
|
48
|
+
after_transition to: :closed do |circuit|
|
38
49
|
circuit.send(:on_circuit_close)
|
39
50
|
end
|
40
51
|
|
41
|
-
after_transition open: :half_open do |circuit|
|
52
|
+
after_transition from: :open, to: :half_open do |circuit|
|
42
53
|
circuit.send(:on_circuit_half_open)
|
43
54
|
end
|
44
55
|
end
|
45
56
|
end
|
46
|
-
|
47
|
-
private
|
48
|
-
|
49
|
-
def on_circuit_open
|
50
|
-
@opened_at.value = BreakerMachines.monotonic_time
|
51
|
-
@storage&.set_status(@name, :open, @opened_at.value)
|
52
|
-
if @storage.respond_to?(:record_event_with_details)
|
53
|
-
@storage.record_event_with_details(@name, :state_change, 0,
|
54
|
-
new_state: :open)
|
55
|
-
end
|
56
|
-
invoke_callback(:on_open)
|
57
|
-
BreakerMachines.instrument('opened', circuit: @name)
|
58
|
-
end
|
59
|
-
|
60
|
-
def on_circuit_close
|
61
|
-
@opened_at.value = nil
|
62
|
-
@last_error.value = nil
|
63
|
-
@last_failure_at.value = nil
|
64
|
-
@storage&.set_status(@name, :closed)
|
65
|
-
if @storage.respond_to?(:record_event_with_details)
|
66
|
-
@storage.record_event_with_details(@name, :state_change, 0,
|
67
|
-
new_state: :closed)
|
68
|
-
end
|
69
|
-
invoke_callback(:on_close)
|
70
|
-
BreakerMachines.instrument('closed', circuit: @name)
|
71
|
-
end
|
72
|
-
|
73
|
-
def on_circuit_half_open
|
74
|
-
@half_open_attempts.value = 0
|
75
|
-
@half_open_successes.value = 0
|
76
|
-
@storage&.set_status(@name, :half_open)
|
77
|
-
if @storage.respond_to?(:record_event_with_details)
|
78
|
-
@storage.record_event_with_details(@name, :state_change, 0,
|
79
|
-
new_state: :half_open)
|
80
|
-
end
|
81
|
-
invoke_callback(:on_half_open)
|
82
|
-
BreakerMachines.instrument('half_opened', circuit: @name)
|
83
|
-
end
|
84
|
-
|
85
|
-
def restore_status_from_storage
|
86
|
-
stored_status = @storage.get_status(@name)
|
87
|
-
return unless stored_status
|
88
|
-
|
89
|
-
self.status = stored_status.status.to_s
|
90
|
-
@opened_at.value = stored_status.opened_at if stored_status.opened_at
|
91
|
-
end
|
92
|
-
|
93
|
-
def reset_timeout_elapsed?
|
94
|
-
return false unless @opened_at.value
|
95
|
-
|
96
|
-
# Add jitter to prevent thundering herd
|
97
|
-
jitter_factor = @config[:reset_timeout_jitter] || 0.25
|
98
|
-
# Calculate random jitter between -jitter_factor and +jitter_factor
|
99
|
-
jitter_multiplier = 1.0 + (((rand * 2) - 1) * jitter_factor)
|
100
|
-
timeout_with_jitter = @config[:reset_timeout] * jitter_multiplier
|
101
|
-
|
102
|
-
BreakerMachines.monotonic_time - @opened_at.value >= timeout_with_jitter
|
103
|
-
end
|
104
57
|
end
|
105
58
|
end
|
106
59
|
end
|
@@ -1,14 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'concurrent-ruby'
|
4
|
-
|
5
3
|
module BreakerMachines
|
6
4
|
class Circuit
|
5
|
+
include Circuit::Base
|
7
6
|
include StateManagement
|
8
|
-
include Configuration
|
9
|
-
include Execution
|
10
|
-
include HedgedExecution
|
11
|
-
include Introspection
|
12
|
-
include Callbacks
|
13
7
|
end
|
14
8
|
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module BreakerMachines
|
4
|
+
# CircuitGroup provides coordinated management of multiple related circuits
|
5
|
+
# with support for dependencies, shared configuration, and group-wide operations
|
6
|
+
class CircuitGroup
|
7
|
+
include BreakerMachines::DSL
|
8
|
+
|
9
|
+
attr_reader :name, :circuits, :dependencies, :config
|
10
|
+
|
11
|
+
def initialize(name, config = {})
|
12
|
+
@name = name
|
13
|
+
@config = config
|
14
|
+
@circuits = {}
|
15
|
+
@dependencies = {}
|
16
|
+
@async_mode = config[:async_mode] || false
|
17
|
+
end
|
18
|
+
|
19
|
+
# Define a circuit within this group with optional dependencies
|
20
|
+
# @param name [Symbol] Circuit name
|
21
|
+
# @param options [Hash] Circuit configuration
|
22
|
+
# @option options [Symbol, Array<Symbol>] :depends_on Other circuits this one depends on
|
23
|
+
# @option options [Proc] :guard_with Additional guard conditions
|
24
|
+
def circuit(name, options = {}, &)
|
25
|
+
depends_on = Array(options.delete(:depends_on))
|
26
|
+
guard_proc = options.delete(:guard_with)
|
27
|
+
|
28
|
+
# Add group-wide defaults
|
29
|
+
circuit_config = @config.merge(options)
|
30
|
+
|
31
|
+
# Create appropriate circuit type
|
32
|
+
circuit_class = if options[:cascades_to] || depends_on.any?
|
33
|
+
BreakerMachines::CascadingCircuit
|
34
|
+
elsif @async_mode
|
35
|
+
BreakerMachines::AsyncCircuit
|
36
|
+
else
|
37
|
+
BreakerMachines::Circuit
|
38
|
+
end
|
39
|
+
|
40
|
+
# Build the circuit
|
41
|
+
circuit_instance = if block_given?
|
42
|
+
builder = BreakerMachines::DSL::CircuitBuilder.new
|
43
|
+
builder.instance_eval(&)
|
44
|
+
built_config = builder.config.merge(circuit_config)
|
45
|
+
circuit_class.new(full_circuit_name(name), built_config)
|
46
|
+
else
|
47
|
+
circuit_class.new(full_circuit_name(name), circuit_config)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Store dependencies and guards
|
51
|
+
if depends_on.any? || guard_proc
|
52
|
+
@dependencies[name] = {
|
53
|
+
depends_on: depends_on,
|
54
|
+
guard: guard_proc
|
55
|
+
}
|
56
|
+
|
57
|
+
# Wrap the circuit with dependency checking
|
58
|
+
circuit_instance = DependencyWrapper.new(circuit_instance, self, name)
|
59
|
+
end
|
60
|
+
|
61
|
+
@circuits[name] = circuit_instance
|
62
|
+
BreakerMachines.register(circuit_instance)
|
63
|
+
circuit_instance
|
64
|
+
end
|
65
|
+
|
66
|
+
# Get a circuit by name
|
67
|
+
def [](name)
|
68
|
+
@circuits[name]
|
69
|
+
end
|
70
|
+
|
71
|
+
# Check if all circuits in the group are healthy
|
72
|
+
def all_healthy?
|
73
|
+
@circuits.values.all? { |circuit| circuit.closed? || circuit.half_open? }
|
74
|
+
end
|
75
|
+
|
76
|
+
# Check if any circuit in the group is open
|
77
|
+
def any_open?
|
78
|
+
@circuits.values.any?(&:open?)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Get status of all circuits
|
82
|
+
def status
|
83
|
+
@circuits.transform_values(&:status_name)
|
84
|
+
end
|
85
|
+
|
86
|
+
# Reset all circuits in the group
|
87
|
+
def reset_all!
|
88
|
+
@circuits.each_value(&:reset!)
|
89
|
+
end
|
90
|
+
|
91
|
+
# Force open all circuits
|
92
|
+
def trip_all!
|
93
|
+
@circuits.each_value(&:force_open!)
|
94
|
+
end
|
95
|
+
|
96
|
+
# Check dependencies for a specific circuit
|
97
|
+
def dependencies_met?(circuit_name)
|
98
|
+
deps = @dependencies[circuit_name]
|
99
|
+
return true unless deps
|
100
|
+
|
101
|
+
depends_on = deps[:depends_on]
|
102
|
+
guard = deps[:guard]
|
103
|
+
|
104
|
+
# Check circuit dependencies recursively
|
105
|
+
dependencies_healthy = depends_on.all? do |dep_name|
|
106
|
+
dep_circuit = @circuits[dep_name]
|
107
|
+
# Circuit must exist, be healthy, AND have its own dependencies met
|
108
|
+
dep_circuit && (dep_circuit.closed? || dep_circuit.half_open?) && dependencies_met?(dep_name)
|
109
|
+
end
|
110
|
+
|
111
|
+
# Check custom guard
|
112
|
+
guard_passed = guard ? guard.call : true
|
113
|
+
|
114
|
+
dependencies_healthy && guard_passed
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def full_circuit_name(name)
|
120
|
+
"#{@name}.#{name}"
|
121
|
+
end
|
122
|
+
|
123
|
+
# Wrapper to enforce dependencies
|
124
|
+
class DependencyWrapper < SimpleDelegator
|
125
|
+
def initialize(circuit, group, name)
|
126
|
+
super(circuit)
|
127
|
+
@group = group
|
128
|
+
@name = name
|
129
|
+
end
|
130
|
+
|
131
|
+
def call(&)
|
132
|
+
unless @group.dependencies_met?(@name)
|
133
|
+
raise BreakerMachines::CircuitDependencyError.new(__getobj__.name,
|
134
|
+
"Dependencies not met for circuit #{@name}")
|
135
|
+
end
|
136
|
+
|
137
|
+
__getobj__.call(&)
|
138
|
+
end
|
139
|
+
|
140
|
+
def attempt_recovery!
|
141
|
+
return false unless @group.dependencies_met?(@name)
|
142
|
+
|
143
|
+
__getobj__.attempt_recovery!
|
144
|
+
end
|
145
|
+
|
146
|
+
def reset!
|
147
|
+
return false unless @group.dependencies_met?(@name)
|
148
|
+
|
149
|
+
__getobj__.reset!
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module BreakerMachines
|
4
|
+
# CoordinatedCircuit is a base class for circuits that need coordinated state management.
|
5
|
+
# It replaces the standard StateManagement module with CoordinatedStateManagement
|
6
|
+
# to enable state transitions based on other circuits' states.
|
7
|
+
class CoordinatedCircuit < Circuit
|
8
|
+
include Circuit::CoordinatedStateManagement
|
9
|
+
end
|
10
|
+
end
|
data/lib/breaker_machines/dsl.rb
CHANGED
@@ -122,7 +122,7 @@ module BreakerMachines
|
|
122
122
|
instance_registry.each do |weak_ref|
|
123
123
|
instance = weak_ref.__getobj__
|
124
124
|
circuit_instances = instance.instance_variable_get(:@circuit_instances)
|
125
|
-
circuit_instances&.each_value(&:
|
125
|
+
circuit_instances&.each_value(&:hard_reset!)
|
126
126
|
rescue WeakRef::RefError
|
127
127
|
# Instance was garbage collected, skip it
|
128
128
|
end
|
@@ -262,7 +262,7 @@ module BreakerMachines
|
|
262
262
|
|
263
263
|
# Reset all circuits for this instance
|
264
264
|
def reset_all_circuits
|
265
|
-
circuit_instances.each_value(&:
|
265
|
+
circuit_instances.each_value(&:hard_reset!)
|
266
266
|
end
|
267
267
|
|
268
268
|
# Remove a global dynamic circuit by name
|
@@ -14,6 +14,16 @@ module BreakerMachines
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
+
# Raised when a circuit cannot be called due to unmet dependencies
|
18
|
+
class CircuitDependencyError < CircuitOpenError
|
19
|
+
def initialize(circuit_name, message = nil)
|
20
|
+
@circuit_name = circuit_name
|
21
|
+
@opened_at = nil
|
22
|
+
super_message = message || "Circuit '#{circuit_name}' cannot be called: dependencies not met"
|
23
|
+
Error.instance_method(:initialize).bind(self).call(super_message)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
17
27
|
# Raised when a circuit-protected call exceeds the configured timeout
|
18
28
|
class CircuitTimeoutError < Error
|
19
29
|
attr_reader :circuit_name, :timeout
|
@@ -48,4 +58,14 @@ module BreakerMachines
|
|
48
58
|
super("Circuit '#{circuit_name}' rejected call: max concurrent limit of #{max_concurrent} reached")
|
49
59
|
end
|
50
60
|
end
|
61
|
+
|
62
|
+
# Raised when all parallel fallbacks fail
|
63
|
+
class ParallelFallbackError < Error
|
64
|
+
attr_reader :errors
|
65
|
+
|
66
|
+
def initialize(message, errors)
|
67
|
+
@errors = errors
|
68
|
+
super(message)
|
69
|
+
end
|
70
|
+
end
|
51
71
|
end
|
@@ -2,10 +2,12 @@
|
|
2
2
|
|
3
3
|
# This file contains async support for hedged execution
|
4
4
|
# It is only loaded when fiber_safe mode is enabled
|
5
|
+
# Requires async gem ~> 2.31.0 for Promise and modern API features
|
5
6
|
|
6
7
|
require 'async'
|
7
8
|
require 'async/task'
|
8
|
-
require 'async/
|
9
|
+
require 'async/promise'
|
10
|
+
require 'async/barrier'
|
9
11
|
require 'concurrent'
|
10
12
|
|
11
13
|
module BreakerMachines
|
@@ -43,53 +45,44 @@ module BreakerMachines
|
|
43
45
|
private
|
44
46
|
|
45
47
|
# Race callables; return first result or raise if it was an Exception
|
46
|
-
# Uses Async::
|
48
|
+
# Uses modern Async::Promise and Async::Barrier for cleaner synchronization
|
47
49
|
# @param callables [Array<Proc>] Tasks to race
|
48
50
|
# @param delay_ms [Integer] Delay in milliseconds between task starts
|
49
51
|
# @return [Object] First successful result
|
50
52
|
# @raise [Exception] The first exception received
|
51
53
|
def race_tasks(callables, delay_ms: 0)
|
52
|
-
|
53
|
-
|
54
|
-
condition = Async::Condition.new
|
55
|
-
winner = nil
|
56
|
-
exception = nil
|
54
|
+
promise = Async::Promise.new
|
55
|
+
barrier = Async::Barrier.new
|
57
56
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
57
|
+
begin
|
58
|
+
result = Async do
|
59
|
+
callables.each_with_index do |callable, idx|
|
60
|
+
barrier.async do
|
61
|
+
# stagger hedged attempts
|
62
|
+
sleep(delay_ms / 1000.0) if idx.positive? && delay_ms.positive?
|
62
63
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
rescue StandardError => e
|
72
|
-
mutex.synchronize do
|
73
|
-
next if winner || exception
|
74
|
-
|
75
|
-
exception = e
|
76
|
-
condition.signal
|
64
|
+
begin
|
65
|
+
result = callable.call
|
66
|
+
# Try to resolve the promise with this result
|
67
|
+
# Only the first resolution will succeed
|
68
|
+
promise.resolve(result) unless promise.resolved?
|
69
|
+
rescue StandardError => e
|
70
|
+
# Only set exception if no result has been resolved yet
|
71
|
+
promise.resolve(e) unless promise.resolved?
|
77
72
|
end
|
78
73
|
end
|
79
74
|
end
|
80
|
-
end
|
81
75
|
|
82
|
-
|
83
|
-
|
76
|
+
# Wait for the first resolution (either success or exception)
|
77
|
+
promise.wait
|
78
|
+
end.wait
|
84
79
|
|
85
|
-
#
|
86
|
-
|
87
|
-
|
88
|
-
#
|
89
|
-
|
90
|
-
|
91
|
-
winner
|
92
|
-
end.wait
|
80
|
+
# If result is an exception, raise it; otherwise return the result
|
81
|
+
result.is_a?(StandardError) ? raise(result) : result
|
82
|
+
ensure
|
83
|
+
# Ensure all tasks are stopped
|
84
|
+
barrier&.stop
|
85
|
+
end
|
93
86
|
end
|
94
87
|
end
|
95
88
|
end
|
data/lib/breaker_machines.rb
CHANGED
@@ -14,6 +14,7 @@ loader.ignore("#{__dir__}/breaker_machines/types.rb")
|
|
14
14
|
loader.ignore("#{__dir__}/breaker_machines/console.rb")
|
15
15
|
loader.ignore("#{__dir__}/breaker_machines/async_support.rb")
|
16
16
|
loader.ignore("#{__dir__}/breaker_machines/hedged_async_support.rb")
|
17
|
+
loader.ignore("#{__dir__}/breaker_machines/circuit/async_state_management.rb")
|
17
18
|
loader.setup
|
18
19
|
|
19
20
|
# BreakerMachines provides a thread-safe implementation of the Circuit Breaker pattern
|
@@ -81,6 +82,22 @@ module BreakerMachines
|
|
81
82
|
Registry.instance
|
82
83
|
end
|
83
84
|
|
85
|
+
# Register a circuit with the global registry
|
86
|
+
def register(circuit)
|
87
|
+
registry.register(circuit)
|
88
|
+
end
|
89
|
+
|
90
|
+
# Reset the registry and configurations (useful for testing)
|
91
|
+
def reset!
|
92
|
+
registry.clear
|
93
|
+
config.default_storage = :bucket_memory
|
94
|
+
config.default_timeout = nil
|
95
|
+
config.default_reset_timeout = 60.seconds
|
96
|
+
config.default_failure_threshold = 5
|
97
|
+
config.log_events = true
|
98
|
+
config.fiber_safe = false
|
99
|
+
end
|
100
|
+
|
84
101
|
# Returns the current monotonic time in seconds.
|
85
102
|
# Monotonic time is guaranteed to always increase and is not affected
|
86
103
|
# by system clock adjustments, making it ideal for measuring durations.
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: breaker_machines
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Abdelkader Boudih
|
@@ -43,14 +43,14 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 0.
|
46
|
+
version: 0.100.0
|
47
47
|
type: :runtime
|
48
48
|
prerelease: false
|
49
49
|
version_requirements: !ruby/object:Gem::Requirement
|
50
50
|
requirements:
|
51
51
|
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 0.
|
53
|
+
version: 0.100.0
|
54
54
|
- !ruby/object:Gem::Dependency
|
55
55
|
name: zeitwerk
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
@@ -108,15 +108,23 @@ files:
|
|
108
108
|
- LICENSE.txt
|
109
109
|
- README.md
|
110
110
|
- lib/breaker_machines.rb
|
111
|
+
- lib/breaker_machines/async_circuit.rb
|
111
112
|
- lib/breaker_machines/async_support.rb
|
112
113
|
- lib/breaker_machines/cascading_circuit.rb
|
113
114
|
- lib/breaker_machines/circuit.rb
|
115
|
+
- lib/breaker_machines/circuit/async_state_management.rb
|
116
|
+
- lib/breaker_machines/circuit/base.rb
|
114
117
|
- lib/breaker_machines/circuit/callbacks.rb
|
115
118
|
- lib/breaker_machines/circuit/configuration.rb
|
119
|
+
- lib/breaker_machines/circuit/coordinated_state_management.rb
|
116
120
|
- lib/breaker_machines/circuit/execution.rb
|
121
|
+
- lib/breaker_machines/circuit/hedged_execution.rb
|
117
122
|
- lib/breaker_machines/circuit/introspection.rb
|
123
|
+
- lib/breaker_machines/circuit/state_callbacks.rb
|
118
124
|
- lib/breaker_machines/circuit/state_management.rb
|
125
|
+
- lib/breaker_machines/circuit_group.rb
|
119
126
|
- lib/breaker_machines/console.rb
|
127
|
+
- lib/breaker_machines/coordinated_circuit.rb
|
120
128
|
- lib/breaker_machines/dsl.rb
|
121
129
|
- lib/breaker_machines/dsl/cascading_circuit_builder.rb
|
122
130
|
- lib/breaker_machines/dsl/circuit_builder.rb
|
@@ -124,7 +132,6 @@ files:
|
|
124
132
|
- lib/breaker_machines/dsl/parallel_fallback_wrapper.rb
|
125
133
|
- lib/breaker_machines/errors.rb
|
126
134
|
- lib/breaker_machines/hedged_async_support.rb
|
127
|
-
- lib/breaker_machines/hedged_execution.rb
|
128
135
|
- lib/breaker_machines/registry.rb
|
129
136
|
- lib/breaker_machines/storage.rb
|
130
137
|
- lib/breaker_machines/storage/backend_state.rb
|
@@ -1,113 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'concurrent'
|
4
|
-
require 'timeout'
|
5
|
-
|
6
|
-
module BreakerMachines
|
7
|
-
# HedgedExecution provides hedged request functionality for circuit breakers
|
8
|
-
# Hedged requests improve latency by sending duplicate requests to multiple backends
|
9
|
-
# and returning the first successful response
|
10
|
-
module HedgedExecution
|
11
|
-
extend ActiveSupport::Concern
|
12
|
-
|
13
|
-
# Execute a hedged request pattern
|
14
|
-
def execute_hedged(&)
|
15
|
-
return execute_single_hedged(&) unless @config[:backends]&.any?
|
16
|
-
|
17
|
-
execute_multi_backend_hedged
|
18
|
-
end
|
19
|
-
|
20
|
-
private
|
21
|
-
|
22
|
-
# Execute hedged request with a single backend (original block)
|
23
|
-
def execute_single_hedged(&block)
|
24
|
-
return yield unless hedged_requests_enabled?
|
25
|
-
|
26
|
-
max_requests = @config[:max_hedged_requests] || 2
|
27
|
-
delay_ms = @config[:hedging_delay] || 50
|
28
|
-
|
29
|
-
if @config[:fiber_safe]
|
30
|
-
execute_hedged_async(Array.new(max_requests) { block }, delay_ms)
|
31
|
-
else
|
32
|
-
execute_hedged_sync(Array.new(max_requests) { block }, delay_ms)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
# Execute hedged requests across multiple backends
|
37
|
-
def execute_multi_backend_hedged
|
38
|
-
backends = @config[:backends]
|
39
|
-
return backends.first.call if backends.size == 1
|
40
|
-
|
41
|
-
if @config[:fiber_safe]
|
42
|
-
execute_hedged_async(backends, @config[:hedging_delay] || 0)
|
43
|
-
else
|
44
|
-
execute_hedged_sync(backends, @config[:hedging_delay] || 0)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
# Synchronous hedged execution using threads
|
49
|
-
def execute_hedged_sync(callables, delay_ms)
|
50
|
-
result_queue = Queue.new
|
51
|
-
error_queue = Queue.new
|
52
|
-
threads = []
|
53
|
-
cancelled = Concurrent::AtomicBoolean.new(false)
|
54
|
-
|
55
|
-
callables.each_with_index do |callable, index|
|
56
|
-
# Add delay for hedge requests (not the first one)
|
57
|
-
sleep(delay_ms / 1000.0) if index.positive? && delay_ms.positive?
|
58
|
-
|
59
|
-
# Skip if already got a result
|
60
|
-
break if cancelled.value
|
61
|
-
|
62
|
-
threads << Thread.new do
|
63
|
-
unless cancelled.value
|
64
|
-
begin
|
65
|
-
result = callable.call
|
66
|
-
result_queue << result unless cancelled.value
|
67
|
-
cancelled.value = true
|
68
|
-
rescue StandardError => e
|
69
|
-
error_queue << e unless cancelled.value
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
# Wait for first result or all errors
|
76
|
-
begin
|
77
|
-
Timeout.timeout(@config[:timeout] || 30) do
|
78
|
-
# Check for successful result
|
79
|
-
loop do
|
80
|
-
unless result_queue.empty?
|
81
|
-
result = result_queue.pop
|
82
|
-
cancelled.value = true
|
83
|
-
return result
|
84
|
-
end
|
85
|
-
|
86
|
-
# Check if all requests failed
|
87
|
-
raise error_queue.pop if error_queue.size >= callables.size
|
88
|
-
|
89
|
-
# Small sleep to prevent busy waiting
|
90
|
-
sleep 0.001
|
91
|
-
end
|
92
|
-
end
|
93
|
-
ensure
|
94
|
-
# Cancel remaining threads
|
95
|
-
cancelled.value = true
|
96
|
-
threads.each(&:kill)
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
# Async hedged execution (requires async support)
|
101
|
-
def execute_hedged_async(callables, delay_ms)
|
102
|
-
# This will be implemented when async support is loaded
|
103
|
-
# For now, fall back to sync implementation
|
104
|
-
return execute_hedged_sync(callables, delay_ms) unless respond_to?(:execute_hedged_with_async)
|
105
|
-
|
106
|
-
execute_hedged_with_async(callables, delay_ms)
|
107
|
-
end
|
108
|
-
|
109
|
-
def hedged_requests_enabled?
|
110
|
-
@config[:hedged_requests] == true
|
111
|
-
end
|
112
|
-
end
|
113
|
-
end
|