faulty 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +78 -1
- data/README.md +22 -2
- data/lib/faulty/cache/auto_wire.rb +2 -2
- data/lib/faulty/cache/circuit_proxy.rb +2 -2
- data/lib/faulty/cache/fault_tolerant_proxy.rb +4 -4
- data/lib/faulty/circuit.rb +39 -7
- data/lib/faulty/error.rb +3 -3
- data/lib/faulty/events/log_listener.rb +11 -2
- data/lib/faulty/events/notifier.rb +3 -5
- data/lib/faulty/immutable_options.rb +4 -4
- data/lib/faulty/patch/base.rb +2 -2
- data/lib/faulty/patch/elasticsearch.rb +2 -2
- data/lib/faulty/patch/redis/patch.rb +1 -1
- data/lib/faulty/patch.rb +2 -2
- data/lib/faulty/status.rb +70 -9
- data/lib/faulty/storage/auto_wire.rb +2 -2
- data/lib/faulty/storage/circuit_proxy.rb +3 -2
- data/lib/faulty/storage/fallback_chain.rb +19 -13
- data/lib/faulty/storage/fault_tolerant_proxy.rb +33 -4
- data/lib/faulty/storage/interface.rb +65 -5
- data/lib/faulty/storage/memory.rb +18 -4
- data/lib/faulty/storage/null.rb +6 -0
- data/lib/faulty/storage/redis.rb +46 -15
- data/lib/faulty/version.rb +1 -1
- data/lib/faulty.rb +13 -11
- metadata +9 -9
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3642cb65b01572d880aec16ece320bed43b9c12ae13a2dce602c1bf2a5a250c3
|
|
4
|
+
data.tar.gz: 7116acd4f458c31a738d0f90a7ac0bc0f37e392e75105f19e7b9648af9d7b4c7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b7078729322061727335102869126e65348f270a71a41b624a2722d62c416672f6b39778338faf570ceac39b7355ebfbc8bf4aff7e866123aada12ba507172ad
|
|
7
|
+
data.tar.gz: 117eae9db002823c62280a7236f5a1969409b6baeffa9c51b0430205f23df5b0238890602cae2217fb19da6e22ea8ebbabf622c750bbb732cd95d0a2caa85554
|
data/CHANGELOG.md
CHANGED
|
@@ -9,6 +9,81 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
9
9
|
[Unreleased]
|
|
10
10
|
-------------------
|
|
11
11
|
|
|
12
|
+
[0.13.0] - 2026-05-13
|
|
13
|
+
---------------------
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
* Reserve half-open test runs across processes so only one process executes
|
|
18
|
+
the test block when a circuit becomes half-open. justinhoward
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
|
|
22
|
+
* `Faulty::Status` now captures `current_time` once when the status is built,
|
|
23
|
+
so all predicates (`open?`, `half_open?`, `reserved?`) reason about the same
|
|
24
|
+
point in time. Previously each predicate called `Faulty.current_time`
|
|
25
|
+
independently. justinhoward
|
|
26
|
+
* `Storage::Redis#close` now also clears the `reserved_at` key. justinhoward
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
|
|
30
|
+
* `Storage::Redis#reserve` uses safe navigation when serializing
|
|
31
|
+
`previous_reserved_at` for `WATCH`, so the very first CAS against a missing
|
|
32
|
+
key compares correctly. justinhoward
|
|
33
|
+
* `Storage::Redis#reset` now clears the `reserved_at` key. justinhoward
|
|
34
|
+
* `Status#can_run?` now treats `locked_closed?` as an unconditional override,
|
|
35
|
+
so a manually locked-closed circuit runs even when a half-open reservation
|
|
36
|
+
is still in effect from a prior cycle. justinhoward
|
|
37
|
+
|
|
38
|
+
### Breaking Changes
|
|
39
|
+
|
|
40
|
+
* `Storage::Interface` adds a required `#reserve(circuit, reserved_at,
|
|
41
|
+
previous_reserved_at)` method. Custom storage backends must implement it,
|
|
42
|
+
and the `Status` value object must carry the new `reserved_at` attribute.
|
|
43
|
+
See `Storage::Interface#reserve` for the contract and the conformance
|
|
44
|
+
test in `spec/storage/interface_spec.rb` for the structural guarantee.
|
|
45
|
+
|
|
46
|
+
[0.12.0] - 2026-05-13
|
|
47
|
+
---------------------
|
|
48
|
+
|
|
49
|
+
Runtime behavior is unchanged from `0.11.0` — the version bump reflects
|
|
50
|
+
support-policy and toolchain breaking changes only. Upgrading should be
|
|
51
|
+
a drop-in replacement on any supported Ruby.
|
|
52
|
+
|
|
53
|
+
### Breaking
|
|
54
|
+
|
|
55
|
+
* Drop support for Ruby < 3.1. Ruby 2.3 – 3.0 are EOL upstream and are no
|
|
56
|
+
longer covered by CI. Faulty now requires Ruby 3.1 or newer.
|
|
57
|
+
* `faulty.gemspec` declares `required_ruby_version = '>= 3.1'`, so older
|
|
58
|
+
Rubies will refuse to install the gem.
|
|
59
|
+
|
|
60
|
+
### Changed
|
|
61
|
+
|
|
62
|
+
* Modernize the CI matrix: Ruby `3.1`, `3.2`, `3.3`, `3.4`, `jruby-head`,
|
|
63
|
+
and `truffleruby-head`; Redis 4 and 5; OpenSearch `2.19.5` (default) and
|
|
64
|
+
`3.0.0`; Elasticsearch `7.17.x` for back-compat coverage. The
|
|
65
|
+
Elasticsearch 7.17 row runs the `elasticsearch:7.17.28` Docker image
|
|
66
|
+
(which ships a JDK that handles cgroupv2 on current runners) against
|
|
67
|
+
the `elasticsearch ~> 7.17.11` gem.
|
|
68
|
+
* Pass `DISABLE_INSTALL_DEMO_CONFIG=true` to the OpenSearch service
|
|
69
|
+
container so OpenSearch 2.12+ doesn't require an admin-password env
|
|
70
|
+
var just to boot up with the security plugin disabled.
|
|
71
|
+
* Replace the deprecated `:mingw` / `:x64_mingw` platform symbols in the
|
|
72
|
+
`Gemfile` with the unified `:windows` symbol Bundler now expects.
|
|
73
|
+
* Upgrade development dependencies: `rubocop ~> 1.84`, `rubocop-rspec ~> 3.9`,
|
|
74
|
+
`simplecov-cobertura ~> 3.1`, `opensearch-ruby ~> 3.4`.
|
|
75
|
+
* `LogListener` now lazy-requires `logger` only when constructing the
|
|
76
|
+
default `Logger.new($stderr)`. Consumers that pass their own logger or
|
|
77
|
+
run under Rails do not need the stdlib `logger` gem on the load path.
|
|
78
|
+
This keeps Faulty boot-clean on Ruby 3.5+ where `logger` is no longer a
|
|
79
|
+
default gem.
|
|
80
|
+
|
|
81
|
+
### Removed
|
|
82
|
+
|
|
83
|
+
* Drop Redis 3 from the test matrix.
|
|
84
|
+
* Drop the Ruby 2.3 carve-out in `spec/spec_helper.rb` that conditionally
|
|
85
|
+
loaded the `mysql2` patch.
|
|
86
|
+
|
|
12
87
|
[0.11.0] - 2023-04-26
|
|
13
88
|
---------------------
|
|
14
89
|
|
|
@@ -308,7 +383,9 @@ of AutoWire.
|
|
|
308
383
|
|
|
309
384
|
Initial public release
|
|
310
385
|
|
|
311
|
-
[Unreleased]: https://github.com/ParentSquare/faulty/compare/v0.
|
|
386
|
+
[Unreleased]: https://github.com/ParentSquare/faulty/compare/v0.13.0...HEAD
|
|
387
|
+
[0.13.0]: https://github.com/ParentSquare/faulty/compare/v0.12.0...v0.13.0
|
|
388
|
+
[0.12.0]: https://github.com/ParentSquare/faulty/compare/v0.11.0...v0.12.0
|
|
312
389
|
[0.11.0]: https://github.com/ParentSquare/faulty/compare/v0.10.0...v0.11.0
|
|
313
390
|
[0.10.0]: https://github.com/ParentSquare/faulty/compare/v0.9.0...v0.10.0
|
|
314
391
|
[0.9.0]: https://github.com/ParentSquare/faulty/compare/v0.8.7...v0.9.0
|
data/README.md
CHANGED
|
@@ -146,7 +146,7 @@ Faulty.init do |config|
|
|
|
146
146
|
config.storage = Faulty::Storage::Redis.new
|
|
147
147
|
|
|
148
148
|
config.listeners << Faulty::Events::CallbackListener.new do |events|
|
|
149
|
-
events.
|
|
149
|
+
events.circuit_opened do |payload|
|
|
150
150
|
puts 'Circuit was opened'
|
|
151
151
|
end
|
|
152
152
|
end
|
|
@@ -1226,6 +1226,23 @@ state, Faulty allows a single execution of the block as a test run. If the test
|
|
|
1226
1226
|
run succeeds, the circuit is fully closed and the circuit state is reset. If the
|
|
1227
1227
|
test run fails, the circuit is opened and the cool-down is reset.
|
|
1228
1228
|
|
|
1229
|
+
When the storage backend supports atomic operations (the default `Memory` and
|
|
1230
|
+
`Redis` backends both do), the half-open test run is reserved exclusively. Other
|
|
1231
|
+
processes or threads that observe the half-open state while a test run is in
|
|
1232
|
+
progress will be skipped with `Faulty::OpenCircuitError`, just as if the circuit
|
|
1233
|
+
were still open. The reservation expires after `cool_down` so that a crashed
|
|
1234
|
+
process can't permanently wedge the circuit.
|
|
1235
|
+
|
|
1236
|
+
This means `cool_down` does double duty: it gates how long the circuit waits
|
|
1237
|
+
before retrying after opening, and it bounds how long a half-open reservation
|
|
1238
|
+
is honored. Test runs that legitimately take longer than `cool_down` (for
|
|
1239
|
+
example, a slow downstream during recovery) will see their reservation expire
|
|
1240
|
+
mid-run, at which point another process can reserve and run the block
|
|
1241
|
+
concurrently. If your protected calls can run longer than `cool_down`, set
|
|
1242
|
+
`cool_down` to comfortably exceed the slowest expected latency for the
|
|
1243
|
+
protected operation, or accept that occasional duplicate half-open test runs
|
|
1244
|
+
are possible during slow recoveries.
|
|
1245
|
+
|
|
1229
1246
|
Each time the circuit changes state or executes the block, events are raised
|
|
1230
1247
|
that are sent to the Faulty event notifier. The notifier should be used to track
|
|
1231
1248
|
circuit failure rates, open circuits, etc.
|
|
@@ -1353,12 +1370,15 @@ but there are and have been many other options:
|
|
|
1353
1370
|
- [circuitbox](https://github.com/yammer/circuitbox): Also uses a block syntax
|
|
1354
1371
|
to manually define circuits. It uses Moneta to abstract circuit storage to
|
|
1355
1372
|
allow any key-value store.
|
|
1373
|
+
- [stoplight](https://github.com/bolshakov/stoplight): Stoplight uses Redis for
|
|
1374
|
+
leader-less coordination in distributed environments to ensure coordinated states
|
|
1375
|
+
transitions and recovery and offers a built-in Admin Panel while focusing on performance
|
|
1376
|
+
by utilizing Lua scripts for the Redis data store which minimizes operational overhead.
|
|
1356
1377
|
|
|
1357
1378
|
### Previous Work
|
|
1358
1379
|
|
|
1359
1380
|
- [circuit_breaker-ruby](https://github.com/scripbox/circuit_breaker-ruby) (no
|
|
1360
1381
|
recent activity)
|
|
1361
|
-
- [stoplight](https://github.com/orgsync/stoplight) (unmaintained)
|
|
1362
1382
|
- [circuit_breaker](https://github.com/wsargent/circuit_breaker) (no recent
|
|
1363
1383
|
activity)
|
|
1364
1384
|
- [simple_circuit_breaker](https://github.com/soundcloud/simple_circuit_breaker)
|
|
@@ -37,8 +37,8 @@ class Faulty
|
|
|
37
37
|
# @param cache [Interface] A cache backend
|
|
38
38
|
# @param options [Hash] Attributes for {Options}
|
|
39
39
|
# @yield [Options] For setting options in a block
|
|
40
|
-
def wrap(cache, **options, &
|
|
41
|
-
options = Options.new(options, &
|
|
40
|
+
def wrap(cache, **options, &)
|
|
41
|
+
options = Options.new(options, &)
|
|
42
42
|
if cache.nil?
|
|
43
43
|
Cache::Default.new
|
|
44
44
|
elsif cache.fault_tolerant?
|
|
@@ -40,9 +40,9 @@ class Faulty
|
|
|
40
40
|
# @param cache [Cache::Interface] The cache backend to wrap
|
|
41
41
|
# @param options [Hash] Attributes for {Options}
|
|
42
42
|
# @yield [Options] For setting options in a block
|
|
43
|
-
def initialize(cache, **options, &
|
|
43
|
+
def initialize(cache, **options, &)
|
|
44
44
|
@cache = cache
|
|
45
|
-
@options = Options.new(options, &
|
|
45
|
+
@options = Options.new(options, &)
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
%i[read write].each do |method|
|
|
@@ -30,19 +30,19 @@ class Faulty
|
|
|
30
30
|
# @param cache [Cache::Interface] The cache backend to wrap
|
|
31
31
|
# @param options [Hash] Attributes for {Options}
|
|
32
32
|
# @yield [Options] For setting options in a block
|
|
33
|
-
def initialize(cache, **options, &
|
|
33
|
+
def initialize(cache, **options, &)
|
|
34
34
|
@cache = cache
|
|
35
|
-
@options = Options.new(options, &
|
|
35
|
+
@options = Options.new(options, &)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# Wrap a cache in a FaultTolerantProxy unless it's already fault tolerant
|
|
39
39
|
#
|
|
40
40
|
# @param cache [Cache::Interface] The cache to maybe wrap
|
|
41
41
|
# @return [Cache::Interface] The original cache or a {FaultTolerantProxy}
|
|
42
|
-
def self.wrap(cache,
|
|
42
|
+
def self.wrap(cache, ...)
|
|
43
43
|
return cache if cache.fault_tolerant?
|
|
44
44
|
|
|
45
|
-
new(cache,
|
|
45
|
+
new(cache, ...)
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
# Read from the cache safely
|
data/lib/faulty/circuit.rb
CHANGED
|
@@ -49,6 +49,10 @@ class Faulty
|
|
|
49
49
|
# @!attribute [r] cool_down
|
|
50
50
|
# @return [Integer] The number of seconds the circuit will
|
|
51
51
|
# stay open after it is tripped. Default 300.
|
|
52
|
+
#
|
|
53
|
+
# Also bounds the half-open reservation TTL — runs longer than
|
|
54
|
+
# `cool_down` lose exclusivity. See the "How it Works" section
|
|
55
|
+
# of the README.
|
|
52
56
|
# @!attribute [r] error_mapper
|
|
53
57
|
# @return [Module, #call] Used by patches to set the namespace module for
|
|
54
58
|
# the faulty errors that will be raised. Should be a module or a callable.
|
|
@@ -178,11 +182,11 @@ class Faulty
|
|
|
178
182
|
# @param name [String] The name of the circuit
|
|
179
183
|
# @param options [Hash] Attributes for {Options}
|
|
180
184
|
# @yield [Options] For setting options in a block
|
|
181
|
-
def initialize(name, **options, &
|
|
185
|
+
def initialize(name, **options, &)
|
|
182
186
|
raise ArgumentError, 'name must be a String' unless name.is_a?(String)
|
|
183
187
|
|
|
184
188
|
@name = name
|
|
185
|
-
@given_options = Options.new(options, &
|
|
189
|
+
@given_options = Options.new(options, &)
|
|
186
190
|
@pulled_options = nil
|
|
187
191
|
@options_pushed = false
|
|
188
192
|
end
|
|
@@ -266,8 +270,8 @@ class Faulty
|
|
|
266
270
|
# @return [Result<Object, Error>] A result where the ok value is the return
|
|
267
271
|
# value of the block, or the error value is an error captured by the
|
|
268
272
|
# circuit.
|
|
269
|
-
def try_run(
|
|
270
|
-
Result.new(ok: run(
|
|
273
|
+
def try_run(...)
|
|
274
|
+
Result.new(ok: run(...))
|
|
271
275
|
rescue FaultyError => e
|
|
272
276
|
Result.new(error: e)
|
|
273
277
|
end
|
|
@@ -308,9 +312,11 @@ class Faulty
|
|
|
308
312
|
return cached_value if !cached_value.nil? && !cache_should_refresh?(cache)
|
|
309
313
|
|
|
310
314
|
current_status = status
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
315
|
+
if current_status.can_run? && reserve(current_status)
|
|
316
|
+
run_exec(current_status, cached_value, cache, &block)
|
|
317
|
+
else
|
|
318
|
+
run_skipped(cached_value)
|
|
319
|
+
end
|
|
314
320
|
end
|
|
315
321
|
|
|
316
322
|
# Force the circuit to stay open until unlocked
|
|
@@ -403,6 +409,32 @@ class Faulty
|
|
|
403
409
|
cached_value
|
|
404
410
|
end
|
|
405
411
|
|
|
412
|
+
# Reserves execution for this circuit when it is half-open
|
|
413
|
+
#
|
|
414
|
+
# This prevents concurrent evaluation from allowing multiple simultaneous
|
|
415
|
+
# runs for half-open circuits. For non-half-open states this is a no-op
|
|
416
|
+
# that returns true so closed and locked-closed circuits run unconditionally.
|
|
417
|
+
#
|
|
418
|
+
# `locked_closed?` is checked before `half_open?` to mirror the operator-
|
|
419
|
+
# override hierarchy in {Status#can_run?}: a locked-closed circuit must
|
|
420
|
+
# always proceed regardless of the underlying state, even if another
|
|
421
|
+
# process is currently holding the reservation. Without this, a locked-
|
|
422
|
+
# closed circuit could lose the storage CAS to a concurrent process and
|
|
423
|
+
# be incorrectly skipped.
|
|
424
|
+
#
|
|
425
|
+
# @param status [Status] The current status of the circuit
|
|
426
|
+
# @return [Boolean] True if this call may proceed to execute the block
|
|
427
|
+
def reserve(status)
|
|
428
|
+
return true if status.locked_closed?
|
|
429
|
+
return true unless status.half_open?
|
|
430
|
+
|
|
431
|
+
# Persist a fresh Faulty.current_time, not status.current_time. The
|
|
432
|
+
# snapshot exists for predicate consistency (see Status#current_time);
|
|
433
|
+
# the stored reserved_at should reflect when the reservation was made,
|
|
434
|
+
# not when the snapshot was taken.
|
|
435
|
+
storage.reserve(self, Faulty.current_time, status.reserved_at)
|
|
436
|
+
end
|
|
437
|
+
|
|
406
438
|
# Execute a run
|
|
407
439
|
#
|
|
408
440
|
# @param cached_value The cached value if one is available
|
data/lib/faulty/error.rb
CHANGED
|
@@ -8,7 +8,7 @@ class Faulty
|
|
|
8
8
|
class UninitializedError < FaultyError
|
|
9
9
|
def initialize(message = nil)
|
|
10
10
|
message ||= 'Faulty is not initialized'
|
|
11
|
-
super
|
|
11
|
+
super
|
|
12
12
|
end
|
|
13
13
|
end
|
|
14
14
|
|
|
@@ -16,7 +16,7 @@ class Faulty
|
|
|
16
16
|
class AlreadyInitializedError < FaultyError
|
|
17
17
|
def initialize(message = nil)
|
|
18
18
|
message ||= 'Faulty is already initialized'
|
|
19
|
-
super
|
|
19
|
+
super
|
|
20
20
|
end
|
|
21
21
|
end
|
|
22
22
|
|
|
@@ -24,7 +24,7 @@ class Faulty
|
|
|
24
24
|
class MissingDefaultInstanceError < FaultyError
|
|
25
25
|
def initialize(message = nil)
|
|
26
26
|
message ||= 'No default instance. Create one with init or get your instance with Faulty[:name]'
|
|
27
|
-
super
|
|
27
|
+
super
|
|
28
28
|
end
|
|
29
29
|
end
|
|
30
30
|
|
|
@@ -10,8 +10,17 @@ class Faulty
|
|
|
10
10
|
# by default if available, otherwise it creates a new `Logger` to
|
|
11
11
|
# stderr.
|
|
12
12
|
def initialize(logger = nil)
|
|
13
|
-
logger
|
|
14
|
-
|
|
13
|
+
@logger = if logger
|
|
14
|
+
logger
|
|
15
|
+
elsif defined?(Rails)
|
|
16
|
+
Rails.logger
|
|
17
|
+
else
|
|
18
|
+
# Lazy-require so consumers who pass their own logger or use Rails
|
|
19
|
+
# don't need the stdlib `logger` gem on the load path. Required for
|
|
20
|
+
# Ruby >= 3.5 where `logger` was extracted from the default gems.
|
|
21
|
+
require 'logger'
|
|
22
|
+
::Logger.new($stderr)
|
|
23
|
+
end
|
|
15
24
|
end
|
|
16
25
|
|
|
17
26
|
# (see ListenerInterface#handle)
|
|
@@ -22,11 +22,9 @@ class Faulty
|
|
|
22
22
|
raise ArgumentError, "Unknown event #{event}" unless EVENTS.include?(event)
|
|
23
23
|
|
|
24
24
|
@listeners.each do |listener|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
warn "Faulty listener #{listener.class.name} crashed: #{e.message}"
|
|
29
|
-
end
|
|
25
|
+
listener.handle(event, payload)
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
warn "Faulty listener #{listener.class.name} crashed: #{e.message}"
|
|
30
28
|
end
|
|
31
29
|
end
|
|
32
30
|
end
|
|
@@ -5,12 +5,12 @@ class Faulty
|
|
|
5
5
|
module ImmutableOptions
|
|
6
6
|
# @param hash [Hash] A hash of attributes to initialize with
|
|
7
7
|
# @yield [self] Yields itself to the block to set options before freezing
|
|
8
|
-
def initialize(hash, &
|
|
9
|
-
setup(defaults.merge(hash), &
|
|
8
|
+
def initialize(hash, &)
|
|
9
|
+
setup(defaults.merge(hash), &)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
def dup_with(hash, &
|
|
13
|
-
dup.setup(hash, &
|
|
12
|
+
def dup_with(hash, &)
|
|
13
|
+
dup.setup(hash, &)
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def setup(hash)
|
data/lib/faulty/patch/base.rb
CHANGED
|
@@ -31,13 +31,13 @@ class Faulty
|
|
|
31
31
|
#
|
|
32
32
|
# @yield A block to run inside the circuit
|
|
33
33
|
# @return The block return value
|
|
34
|
-
def faulty_run(&
|
|
34
|
+
def faulty_run(&)
|
|
35
35
|
faulty_running_key = "faulty_running_#{object_id}"
|
|
36
36
|
return yield unless @faulty_circuit
|
|
37
37
|
return yield if Thread.current[faulty_running_key]
|
|
38
38
|
|
|
39
39
|
Thread.current[faulty_running_key] = true
|
|
40
|
-
@faulty_circuit.run(&
|
|
40
|
+
@faulty_circuit.run(&)
|
|
41
41
|
ensure
|
|
42
42
|
Thread.current[faulty_running_key] = nil
|
|
43
43
|
end
|
|
@@ -54,7 +54,7 @@ class Faulty
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
module Errors
|
|
57
|
-
PATCHED_MODULE::Transport::Transport::ERRORS.
|
|
57
|
+
PATCHED_MODULE::Transport::Transport::ERRORS.each_value do |klass|
|
|
58
58
|
MAPPED_ERRORS[klass] = const_set(klass.name.split('::').last, Module.new)
|
|
59
59
|
end
|
|
60
60
|
end
|
|
@@ -69,7 +69,7 @@ class Faulty
|
|
|
69
69
|
end
|
|
70
70
|
private_constant :ERROR_MAPPER, :MAPPED_ERRORS
|
|
71
71
|
|
|
72
|
-
def initialize(arguments = {}, &
|
|
72
|
+
def initialize(arguments = {}, &)
|
|
73
73
|
super
|
|
74
74
|
|
|
75
75
|
errors = [PATCHED_MODULE::Transport::Transport::Error]
|
data/lib/faulty/patch.rb
CHANGED
|
@@ -75,7 +75,7 @@ class Faulty
|
|
|
75
75
|
# `:error_mapper`
|
|
76
76
|
# @yield [Circuit::Options] For setting override options in a block
|
|
77
77
|
# @return [Circuit, nil] The circuit if one was created
|
|
78
|
-
def circuit_from_hash(default_name, hash, **options, &
|
|
78
|
+
def circuit_from_hash(default_name, hash, **options, &)
|
|
79
79
|
return unless hash
|
|
80
80
|
|
|
81
81
|
hash = symbolize_keys(hash)
|
|
@@ -84,7 +84,7 @@ class Faulty
|
|
|
84
84
|
error_mapper = options.delete(:patched_error_mapper)
|
|
85
85
|
hash[:error_mapper] ||= error_mapper if error_mapper && patch_errors
|
|
86
86
|
faulty = resolve_instance(hash.delete(:instance))
|
|
87
|
-
faulty.circuit(name, **hash, **options, &
|
|
87
|
+
faulty.circuit(name, **hash, **options, &)
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
# Create a full set of {CircuitError}s with a given base error class
|
data/lib/faulty/status.rb
CHANGED
|
@@ -16,9 +16,19 @@ class Faulty
|
|
|
16
16
|
# @return [:open, :closed, nil] If the circuit is locked, the state that
|
|
17
17
|
# it is locked in. Default `nil`.
|
|
18
18
|
# @!attribute [r] opened_at
|
|
19
|
-
# @return [
|
|
20
|
-
# opened. This is not necessarily reset when the circuit
|
|
21
|
-
# Default `nil`.
|
|
19
|
+
# @return [Float, nil] If the circuit is open, the timestamp ({Faulty.current_time})
|
|
20
|
+
# that it was opened. This is not necessarily reset when the circuit
|
|
21
|
+
# is closed. Default `nil`.
|
|
22
|
+
# @!attribute [r] reserved_at
|
|
23
|
+
# @return [Float, nil] If a half-open test run was reserved, the
|
|
24
|
+
# timestamp ({Faulty.current_time}) of that reservation. Cleared when
|
|
25
|
+
# the circuit is closed.
|
|
26
|
+
# Not reset by {Storage::Interface#reopen}; the value naturally expires
|
|
27
|
+
# via `cool_down`. Default `nil`.
|
|
28
|
+
#
|
|
29
|
+
# Only meaningful when {#state} is `:open`. If a backend race or bug
|
|
30
|
+
# produces an inconsistent shape (`state == :closed` with a non-nil
|
|
31
|
+
# `reserved_at`), it is normalized to `nil` at construction.
|
|
22
32
|
# @!attribute [r] failure_rate
|
|
23
33
|
# @return [Float] A number from 0 to 1 representing the percentage of
|
|
24
34
|
# failures for the circuit. For exmaple 0.5 represents a 50% failure rate.
|
|
@@ -34,6 +44,7 @@ class Faulty
|
|
|
34
44
|
:state,
|
|
35
45
|
:lock,
|
|
36
46
|
:opened_at,
|
|
47
|
+
:reserved_at,
|
|
37
48
|
:failure_rate,
|
|
38
49
|
:sample_size,
|
|
39
50
|
:options,
|
|
@@ -43,6 +54,19 @@ class Faulty
|
|
|
43
54
|
class Status
|
|
44
55
|
include ImmutableOptions
|
|
45
56
|
|
|
57
|
+
# @return [Float] The point in time captured when this status was built.
|
|
58
|
+
# All predicates (`open?`, `half_open?`, `reserved?`) reason about
|
|
59
|
+
# this same instant so they are mutually consistent. Held as an
|
|
60
|
+
# instance variable rather than a struct field so it does not leak
|
|
61
|
+
# into `to_h`, `==`, or `members` — those should reflect persisted
|
|
62
|
+
# circuit state, not a transient predicate-consistency snapshot.
|
|
63
|
+
attr_reader :current_time
|
|
64
|
+
|
|
65
|
+
def initialize(hash, &)
|
|
66
|
+
@current_time = hash[:current_time] || Faulty.current_time
|
|
67
|
+
super(hash.except(:current_time), &)
|
|
68
|
+
end
|
|
69
|
+
|
|
46
70
|
# The allowed state values
|
|
47
71
|
STATES = %i[
|
|
48
72
|
open
|
|
@@ -66,7 +90,8 @@ class Faulty
|
|
|
66
90
|
# sample_size
|
|
67
91
|
# @return [Status]
|
|
68
92
|
def self.from_entries(entries, **hash)
|
|
69
|
-
|
|
93
|
+
current_time = Faulty.current_time
|
|
94
|
+
window_start = current_time - hash[:options].evaluation_window
|
|
70
95
|
size = entries.size
|
|
71
96
|
i = 0
|
|
72
97
|
failures = 0
|
|
@@ -84,7 +109,8 @@ class Faulty
|
|
|
84
109
|
|
|
85
110
|
new(hash.merge(
|
|
86
111
|
sample_size: sample_size,
|
|
87
|
-
failure_rate: sample_size.zero? ? 0.0 : failures.to_f / sample_size
|
|
112
|
+
failure_rate: sample_size.zero? ? 0.0 : failures.to_f / sample_size,
|
|
113
|
+
current_time: current_time
|
|
88
114
|
))
|
|
89
115
|
end
|
|
90
116
|
|
|
@@ -94,7 +120,7 @@ class Faulty
|
|
|
94
120
|
#
|
|
95
121
|
# @return [Boolean] True if open
|
|
96
122
|
def open?
|
|
97
|
-
state == :open && opened_at + options.cool_down >
|
|
123
|
+
state == :open && opened_at + options.cool_down > current_time
|
|
98
124
|
end
|
|
99
125
|
|
|
100
126
|
# Whether the circuit is closed
|
|
@@ -112,7 +138,7 @@ class Faulty
|
|
|
112
138
|
#
|
|
113
139
|
# @return [Boolean] True if half-open
|
|
114
140
|
def half_open?
|
|
115
|
-
state == :open && opened_at + options.cool_down <=
|
|
141
|
+
state == :open && opened_at + options.cool_down <= current_time
|
|
116
142
|
end
|
|
117
143
|
|
|
118
144
|
# Whether the circuit is locked open
|
|
@@ -129,15 +155,42 @@ class Faulty
|
|
|
129
155
|
lock == :closed
|
|
130
156
|
end
|
|
131
157
|
|
|
158
|
+
# Whether a half-open test run is currently reserved
|
|
159
|
+
#
|
|
160
|
+
# Process-agnostic: returns true whenever an unexpired reservation exists
|
|
161
|
+
# on this circuit, regardless of who made it. The "did someone else reserve
|
|
162
|
+
# this?" interpretation only applies when this predicate is read on a
|
|
163
|
+
# status snapshot taken *before* the caller attempts {Storage::Interface#reserve};
|
|
164
|
+
# a caller introspecting their own status after a successful reserve will
|
|
165
|
+
# also see `true` here.
|
|
166
|
+
#
|
|
167
|
+
# The reservation expires after `cool_down` to handle the case where the
|
|
168
|
+
# process that made the reservation crashes before resolving the circuit.
|
|
169
|
+
# Side effect: a legitimately-slow test run that exceeds `cool_down`
|
|
170
|
+
# loses exclusivity (another process may reserve and run concurrently).
|
|
171
|
+
# See the "How it Works" section of the README for the full trade-off.
|
|
172
|
+
#
|
|
173
|
+
# @return [Boolean] True if a reservation is in effect
|
|
174
|
+
def reserved?
|
|
175
|
+
return false unless reserved_at
|
|
176
|
+
|
|
177
|
+
state == :open && reserved_at + options.cool_down > current_time
|
|
178
|
+
end
|
|
179
|
+
|
|
132
180
|
# Whether the circuit can be run
|
|
133
181
|
#
|
|
134
|
-
# Takes the circuit state, locks and cooldown into account
|
|
182
|
+
# Takes the circuit state, locks and cooldown into account. Locks are
|
|
183
|
+
# operator overrides and take precedence over both state and reservation,
|
|
184
|
+
# so a `locked_closed?` circuit always runs and a `locked_open?` circuit
|
|
185
|
+
# never runs.
|
|
135
186
|
#
|
|
136
187
|
# @return [Boolean] True if the circuit can be run
|
|
137
188
|
def can_run?
|
|
138
189
|
return false if locked_open?
|
|
190
|
+
return true if locked_closed?
|
|
191
|
+
return false if reserved?
|
|
139
192
|
|
|
140
|
-
closed? ||
|
|
193
|
+
closed? || half_open?
|
|
141
194
|
end
|
|
142
195
|
|
|
143
196
|
# Whether the circuit fails the sample size and rate thresholds
|
|
@@ -155,6 +208,14 @@ class Faulty
|
|
|
155
208
|
raise ArgumentError, "lock must be a symbol in #{self.class}::LOCKS or nil"
|
|
156
209
|
end
|
|
157
210
|
raise ArgumentError, 'opened_at is required if state is open' if state == :open && opened_at.nil?
|
|
211
|
+
|
|
212
|
+
# `reserved_at` is only meaningful while the circuit is open. Backends
|
|
213
|
+
# are expected to clear it on close, but if a brief race or backend bug
|
|
214
|
+
# leaves a stale value paired with `state == :closed`, normalize it
|
|
215
|
+
# here so downstream code can rely on the invariant without checking
|
|
216
|
+
# `state` first. Sanitizing rather than raising avoids turning a
|
|
217
|
+
# transient backend inconsistency into a production crash.
|
|
218
|
+
self.reserved_at = nil if state == :closed && !reserved_at.nil?
|
|
158
219
|
end
|
|
159
220
|
|
|
160
221
|
def required
|
|
@@ -52,8 +52,8 @@ class Faulty
|
|
|
52
52
|
# of storage backends to setup.
|
|
53
53
|
# @param options [Hash] Attributes for {Options}
|
|
54
54
|
# @yield [Options] For setting options in a block
|
|
55
|
-
def wrap(storage, **options, &
|
|
56
|
-
options = Options.new(options, &
|
|
55
|
+
def wrap(storage, **options, &)
|
|
56
|
+
options = Options.new(options, &)
|
|
57
57
|
if storage.nil?
|
|
58
58
|
Memory.new
|
|
59
59
|
elsif storage.is_a?(Array)
|
|
@@ -40,9 +40,9 @@ class Faulty
|
|
|
40
40
|
# @param storage [Storage::Interface] The storage backend to wrap
|
|
41
41
|
# @param options [Hash] Attributes for {Options}
|
|
42
42
|
# @yield [Options] For setting options in a block
|
|
43
|
-
def initialize(storage, **options, &
|
|
43
|
+
def initialize(storage, **options, &)
|
|
44
44
|
@storage = storage
|
|
45
|
-
@options = Options.new(options, &
|
|
45
|
+
@options = Options.new(options, &)
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
%i[
|
|
@@ -52,6 +52,7 @@ class Faulty
|
|
|
52
52
|
open
|
|
53
53
|
reopen
|
|
54
54
|
close
|
|
55
|
+
reserve
|
|
55
56
|
lock
|
|
56
57
|
unlock
|
|
57
58
|
reset
|
|
@@ -42,9 +42,9 @@ class Faulty
|
|
|
42
42
|
# additional entries will be tried in sequence until one succeeds.
|
|
43
43
|
# @param options [Hash] Attributes for {Options}
|
|
44
44
|
# @yield [Options] For setting options in a block
|
|
45
|
-
def initialize(storages, **options, &
|
|
45
|
+
def initialize(storages, **options, &)
|
|
46
46
|
@storages = storages
|
|
47
|
-
@options = Options.new(options, &
|
|
47
|
+
@options = Options.new(options, &)
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
# Get options from the first available storage backend
|
|
@@ -105,6 +105,16 @@ class Faulty
|
|
|
105
105
|
end
|
|
106
106
|
end
|
|
107
107
|
|
|
108
|
+
# Reserve a half-open run in the first available storage backend
|
|
109
|
+
#
|
|
110
|
+
# @param (see Interface#reserve)
|
|
111
|
+
# @return (see Interface#reserve)
|
|
112
|
+
def reserve(circuit, reserved_at, previous_reserved_at)
|
|
113
|
+
send_chain(:reserve, circuit, reserved_at, previous_reserved_at) do |e|
|
|
114
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :reserve, error: e)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
108
118
|
# Lock a circuit in all storage backends
|
|
109
119
|
#
|
|
110
120
|
# @param (see Interface#lock)
|
|
@@ -189,12 +199,10 @@ class Faulty
|
|
|
189
199
|
def send_chain(method, *args)
|
|
190
200
|
errors = []
|
|
191
201
|
@storages.each do |s|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
yield e
|
|
197
|
-
end
|
|
202
|
+
return s.public_send(method, *args)
|
|
203
|
+
rescue StandardError => e
|
|
204
|
+
errors << e
|
|
205
|
+
yield e
|
|
198
206
|
end
|
|
199
207
|
|
|
200
208
|
raise AllFailedError.new("#{self.class}##{method} failed for all storage backends", errors)
|
|
@@ -211,11 +219,9 @@ class Faulty
|
|
|
211
219
|
def send_all(method, *args)
|
|
212
220
|
errors = []
|
|
213
221
|
@storages.each do |s|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
errors << e
|
|
218
|
-
end
|
|
222
|
+
s.public_send(method, *args)
|
|
223
|
+
rescue StandardError => e
|
|
224
|
+
errors << e
|
|
219
225
|
end
|
|
220
226
|
|
|
221
227
|
if errors.empty?
|
|
@@ -9,6 +9,19 @@ class Faulty
|
|
|
9
9
|
#
|
|
10
10
|
# If the storage backend raises a `StandardError`, it will be captured and
|
|
11
11
|
# sent to the notifier.
|
|
12
|
+
#
|
|
13
|
+
# The overall design preference is to keep protected code paths running
|
|
14
|
+
# when the storage backend is degraded, even when that means losing
|
|
15
|
+
# circuit-breaker protections that the storage normally provides:
|
|
16
|
+
# `#status` returns a stub closed status (so `Circuit#run` proceeds),
|
|
17
|
+
# `#reserve` returns `true` (so half-open test runs proceed), and the
|
|
18
|
+
# write paths (`#open`, `#reopen`, `#close`, `#entry`) return `false`
|
|
19
|
+
# to safe-deny the *recorded transition* without failing the in-flight
|
|
20
|
+
# call. The trade-off is that a correlated outage of the storage
|
|
21
|
+
# backend and the upstream protected by the circuit will let the fleet
|
|
22
|
+
# converge on the upstream — but that fleet would converge anyway via
|
|
23
|
+
# the stub-closed status path, so individual write methods don't make
|
|
24
|
+
# it worse.
|
|
12
25
|
class FaultTolerantProxy
|
|
13
26
|
extend Forwardable
|
|
14
27
|
|
|
@@ -31,9 +44,9 @@ class Faulty
|
|
|
31
44
|
# @param storage [Storage::Interface] The storage backend to wrap
|
|
32
45
|
# @param options [Hash] Attributes for {Options}
|
|
33
46
|
# @yield [Options] For setting options in a block
|
|
34
|
-
def initialize(storage, **options, &
|
|
47
|
+
def initialize(storage, **options, &)
|
|
35
48
|
@storage = storage
|
|
36
|
-
@options = Options.new(options, &
|
|
49
|
+
@options = Options.new(options, &)
|
|
37
50
|
end
|
|
38
51
|
|
|
39
52
|
# Wrap a storage backend in a FaultTolerantProxy unless it's already
|
|
@@ -41,10 +54,10 @@ class Faulty
|
|
|
41
54
|
#
|
|
42
55
|
# @param storage [Storage::Interface] The storage to maybe wrap
|
|
43
56
|
# @return [Storage::Interface] The original storage or a {FaultTolerantProxy}
|
|
44
|
-
def self.wrap(storage,
|
|
57
|
+
def self.wrap(storage, ...)
|
|
45
58
|
return storage if storage.fault_tolerant?
|
|
46
59
|
|
|
47
|
-
new(storage,
|
|
60
|
+
new(storage, ...)
|
|
48
61
|
end
|
|
49
62
|
|
|
50
63
|
# @!method lock(circuit, state)
|
|
@@ -177,6 +190,22 @@ class Faulty
|
|
|
177
190
|
stub_status(circuit)
|
|
178
191
|
end
|
|
179
192
|
|
|
193
|
+
# Safely reserve execution of a circuit
|
|
194
|
+
#
|
|
195
|
+
# Returns `true` on storage error so half-open test runs proceed when
|
|
196
|
+
# the backend is degraded. See the class-level docs for the gem's
|
|
197
|
+
# fail-open trade-off.
|
|
198
|
+
#
|
|
199
|
+
# @see Interface#reserve
|
|
200
|
+
# @param (see Interface#reserve)
|
|
201
|
+
# @return (see Interface#reserve)
|
|
202
|
+
def reserve(circuit, reserved_at, previous_reserved_at)
|
|
203
|
+
@storage.reserve(circuit, reserved_at, previous_reserved_at)
|
|
204
|
+
rescue StandardError => e
|
|
205
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :reserve, error: e)
|
|
206
|
+
true
|
|
207
|
+
end
|
|
208
|
+
|
|
180
209
|
# This cache makes any storage fault tolerant, so this is always `true`
|
|
181
210
|
#
|
|
182
211
|
# @return [true]
|
|
@@ -35,7 +35,8 @@ class Faulty
|
|
|
35
35
|
# long as it can implement the other read methods.
|
|
36
36
|
#
|
|
37
37
|
# @param circuit [Circuit] The circuit that ran
|
|
38
|
-
# @param time [
|
|
38
|
+
# @param time [Float] The unix timestamp for the run, from
|
|
39
|
+
# {Faulty.current_time}
|
|
39
40
|
# @param success [Boolean] True if the run succeeded
|
|
40
41
|
# @param status [Status, nil] The previous status. If given, this method must
|
|
41
42
|
# return an updated status object from the new entry data.
|
|
@@ -58,7 +59,8 @@ class Faulty
|
|
|
58
59
|
# current time.
|
|
59
60
|
#
|
|
60
61
|
# @param circuit [Circuit] The circuit to open
|
|
61
|
-
# @param opened_at [
|
|
62
|
+
# @param opened_at [Float] The timestamp the circuit was opened at,
|
|
63
|
+
# from {Faulty.current_time}
|
|
62
64
|
# @return [Boolean] True if the circuit transitioned from closed to open
|
|
63
65
|
def open(circuit, opened_at)
|
|
64
66
|
raise NotImplementedError
|
|
@@ -75,10 +77,35 @@ class Faulty
|
|
|
75
77
|
# it may always return true, but that could result in duplicate reopen
|
|
76
78
|
# notifications.
|
|
77
79
|
#
|
|
80
|
+
# The backend MUST NOT clear `reserved_at` here.
|
|
81
|
+
#
|
|
82
|
+
# Preserving the prior cycle's `reserved_at` is load-bearing for
|
|
83
|
+
# half-open exclusivity. If a late-arriving caller read status while
|
|
84
|
+
# `reserved_at` was still nil (before the winning process reserved),
|
|
85
|
+
# its subsequent `reserve(circuit, T, nil)` CAS must fail. Clearing
|
|
86
|
+
# `reserved_at` in `reopen` would let that stale CAS incorrectly
|
|
87
|
+
# succeed and produce a duplicate half-open run.
|
|
88
|
+
#
|
|
89
|
+
# Beyond exclusivity, the prior reservation expires naturally at
|
|
90
|
+
# `reserved_at + cool_down`, aligned with the start of the next
|
|
91
|
+
# half-open window (since `reserved_at <= new_opened_at`). An
|
|
92
|
+
# explicit reset would be redundant with the cool-down-aligned expiry
|
|
93
|
+
# that already handles crash recovery.
|
|
94
|
+
#
|
|
95
|
+
# This invariant assumes the reservation TTL equals `cool_down`. If
|
|
96
|
+
# a separate `reservation_ttl` is ever introduced, this method must
|
|
97
|
+
# be revisited and may need to clear `reserved_at` explicitly.
|
|
98
|
+
#
|
|
78
99
|
# @param circuit [Circuit] The circuit to reopen
|
|
79
|
-
# @param opened_at [
|
|
80
|
-
#
|
|
81
|
-
#
|
|
100
|
+
# @param opened_at [Float] The timestamp the circuit was opened at,
|
|
101
|
+
# from {Faulty.current_time}
|
|
102
|
+
# @param previous_opened_at [Float] The last known value of opened_at.
|
|
103
|
+
# Can be used to compare-and-set. Always non-nil — `Circuit#failure!`
|
|
104
|
+
# only enters the reopen branch when `status.half_open?` is true,
|
|
105
|
+
# which requires non-nil `opened_at`. Unlike `previous_reserved_at`
|
|
106
|
+
# on {#reserve}, there is no legitimate "no prior value" call path
|
|
107
|
+
# to `reopen`, so backends may treat this parameter as required and
|
|
108
|
+
# are not expected to handle `nil`.
|
|
82
109
|
# @return [Boolean] True if the opened_at time was updated
|
|
83
110
|
def reopen(circuit, opened_at, previous_opened_at)
|
|
84
111
|
raise NotImplementedError
|
|
@@ -90,6 +117,9 @@ class Faulty
|
|
|
90
117
|
# may be called more than once. If so, this method should return true
|
|
91
118
|
# only once, when the circuit transitions from open to closed.
|
|
92
119
|
#
|
|
120
|
+
# The backend should reset the reserved_at value to empty when closing
|
|
121
|
+
# the circuit.
|
|
122
|
+
#
|
|
93
123
|
# If the backend does not support locking or atomic operations, then
|
|
94
124
|
# it may always return true, but that could result in duplicate close
|
|
95
125
|
# notifications.
|
|
@@ -99,6 +129,36 @@ class Faulty
|
|
|
99
129
|
raise NotImplementedError
|
|
100
130
|
end
|
|
101
131
|
|
|
132
|
+
# Reserve an exclusive run for this circuit
|
|
133
|
+
#
|
|
134
|
+
# This is used when the circuit is half-open and the test run is being
|
|
135
|
+
# attempted. We need to make sure only a single run is allowed.
|
|
136
|
+
#
|
|
137
|
+
# The backend should store reserved_at and use it to serve future status
|
|
138
|
+
# requests. When setting reserved_at, the backend should atomically
|
|
139
|
+
# compare any existing value using previous_reserved_at. This ensures
|
|
140
|
+
# that mutltiple parallel processes can't reserve the circuit.
|
|
141
|
+
#
|
|
142
|
+
# The return value is the caller's signal to proceed with the half-open
|
|
143
|
+
# test run, not a strict report of whether atomic acquisition succeeded.
|
|
144
|
+
# Atomic backends should return `true` only when the CAS against
|
|
145
|
+
# `previous_reserved_at` succeeds. Non-atomic backends, no-op backends
|
|
146
|
+
# ({Null}), or wrappers that fail open ({FaultTolerantProxy}) may always
|
|
147
|
+
# return `true`; the caller will proceed at the cost of allowing
|
|
148
|
+
# duplicate half-open test runs.
|
|
149
|
+
#
|
|
150
|
+
# @param circuit [Circuit] The circuit to reserve
|
|
151
|
+
# @param reserved_at [Float] The timestamp of this reservation, from
|
|
152
|
+
# {Faulty.current_time}
|
|
153
|
+
# @param previous_reserved_at [Float, nil] The last known value of
|
|
154
|
+
# reserved_at, or nil for the first reservation in a new open cycle.
|
|
155
|
+
# Can be used to compare-and-set.
|
|
156
|
+
# @return [Boolean] True if the caller may proceed with the half-open
|
|
157
|
+
# test run; false if another caller already holds the reservation.
|
|
158
|
+
def reserve(circuit, reserved_at, previous_reserved_at)
|
|
159
|
+
raise NotImplementedError
|
|
160
|
+
end
|
|
161
|
+
|
|
102
162
|
# Lock the circuit in a given state
|
|
103
163
|
#
|
|
104
164
|
# No concurrency gurantees are provided for locking
|
|
@@ -41,11 +41,12 @@ class Faulty
|
|
|
41
41
|
# The internal object for storing a circuit
|
|
42
42
|
#
|
|
43
43
|
# @private
|
|
44
|
-
MemoryCircuit = Struct.new(:state, :runs, :opened_at, :lock, :options) do
|
|
44
|
+
MemoryCircuit = Struct.new(:state, :runs, :opened_at, :reserved_at, :lock, :options) do
|
|
45
45
|
def initialize
|
|
46
46
|
self.state = Concurrent::Atom.new(:closed)
|
|
47
47
|
self.runs = Concurrent::MVar.new([], dup_on_deref: true)
|
|
48
48
|
self.opened_at = Concurrent::Atom.new(nil)
|
|
49
|
+
self.reserved_at = Concurrent::Atom.new(nil)
|
|
49
50
|
self.lock = nil
|
|
50
51
|
end
|
|
51
52
|
|
|
@@ -61,6 +62,7 @@ class Faulty
|
|
|
61
62
|
state: state.value,
|
|
62
63
|
lock: lock,
|
|
63
64
|
opened_at: opened_at.value,
|
|
65
|
+
reserved_at: reserved_at.value,
|
|
64
66
|
options: circuit_options
|
|
65
67
|
)
|
|
66
68
|
end
|
|
@@ -71,9 +73,9 @@ class Faulty
|
|
|
71
73
|
|
|
72
74
|
# @param options [Hash] Attributes for {Options}
|
|
73
75
|
# @yield [Options] For setting options in a block
|
|
74
|
-
def initialize(**options, &
|
|
76
|
+
def initialize(**options, &)
|
|
75
77
|
@circuits = Concurrent::Map.new
|
|
76
|
-
@options = Options.new(options, &
|
|
78
|
+
@options = Options.new(options, &)
|
|
77
79
|
end
|
|
78
80
|
|
|
79
81
|
# Get the options stored for circuit
|
|
@@ -139,7 +141,19 @@ class Faulty
|
|
|
139
141
|
def close(circuit)
|
|
140
142
|
memory = fetch(circuit)
|
|
141
143
|
memory.runs.modify { |_old| [] }
|
|
142
|
-
memory.state.compare_and_set(:open, :closed)
|
|
144
|
+
closed = memory.state.compare_and_set(:open, :closed)
|
|
145
|
+
memory.reserved_at.reset(nil) if closed
|
|
146
|
+
closed
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Reserve an exclusive run for this circuit
|
|
150
|
+
#
|
|
151
|
+
# @see Interface#reserve
|
|
152
|
+
# @param (see Interface#reserve)
|
|
153
|
+
# @return (see Interface#reserve)
|
|
154
|
+
def reserve(circuit, reserved_at, previous_reserved_at)
|
|
155
|
+
memory = fetch(circuit)
|
|
156
|
+
memory.reserved_at.compare_and_set(previous_reserved_at, reserved_at)
|
|
143
157
|
end
|
|
144
158
|
|
|
145
159
|
# Lock a circuit open or closed
|
data/lib/faulty/storage/null.rb
CHANGED
|
@@ -46,6 +46,12 @@ class Faulty
|
|
|
46
46
|
true
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
# @param (see Interface#reserve)
|
|
50
|
+
# @return (see Interface#reserve)
|
|
51
|
+
def reserve(_circuit, _reserved_at, _previous_reserved_at)
|
|
52
|
+
true
|
|
53
|
+
end
|
|
54
|
+
|
|
49
55
|
# @param (see Interface#lock)
|
|
50
56
|
# @return (see Interface#lock)
|
|
51
57
|
def lock(_circuit, _state)
|
data/lib/faulty/storage/redis.rb
CHANGED
|
@@ -80,8 +80,8 @@ class Faulty
|
|
|
80
80
|
|
|
81
81
|
# @param options [Hash] Attributes for {Options}
|
|
82
82
|
# @yield [Options] For setting options in a block
|
|
83
|
-
def initialize(**options, &
|
|
84
|
-
@options = Options.new(options, &
|
|
83
|
+
def initialize(**options, &)
|
|
84
|
+
@options = Options.new(options, &)
|
|
85
85
|
|
|
86
86
|
# Ensure JSON is available since we don't explicitly require it
|
|
87
87
|
JSON # rubocop:disable Lint/Void
|
|
@@ -174,11 +174,25 @@ class Faulty
|
|
|
174
174
|
result = watch_exec(key, ['open']) do |m|
|
|
175
175
|
m.set(key, 'closed', ex: ex)
|
|
176
176
|
m.del(entries_key(circuit.name))
|
|
177
|
+
m.del(reserved_at_key(circuit.name))
|
|
177
178
|
end
|
|
178
179
|
|
|
179
180
|
result && result[0] == 'OK'
|
|
180
181
|
end
|
|
181
182
|
|
|
183
|
+
# Reserve an exclusive run for this circuit
|
|
184
|
+
#
|
|
185
|
+
# @see Interface#reserve
|
|
186
|
+
# @param (see Interface#reserve)
|
|
187
|
+
# @return (see Interface#reserve)
|
|
188
|
+
def reserve(circuit, reserved_at, previous_reserved_at)
|
|
189
|
+
key = reserved_at_key(circuit.name)
|
|
190
|
+
result = watch_exec(key, [previous_reserved_at&.to_s]) do |m|
|
|
191
|
+
m.set(key, reserved_at, ex: options.circuit_ttl)
|
|
192
|
+
end
|
|
193
|
+
result && result[0] == 'OK'
|
|
194
|
+
end
|
|
195
|
+
|
|
182
196
|
# Lock a circuit open or closed
|
|
183
197
|
#
|
|
184
198
|
# The circuit_ttl does not apply to locks
|
|
@@ -210,6 +224,7 @@ class Faulty
|
|
|
210
224
|
r.del(
|
|
211
225
|
entries_key(name),
|
|
212
226
|
opened_at_key(name),
|
|
227
|
+
reserved_at_key(name),
|
|
213
228
|
lock_key(name),
|
|
214
229
|
options_key(name)
|
|
215
230
|
)
|
|
@@ -228,20 +243,11 @@ class Faulty
|
|
|
228
243
|
futures[:state] = r.get(state_key(circuit.name))
|
|
229
244
|
futures[:lock] = r.get(lock_key(circuit.name))
|
|
230
245
|
futures[:opened_at] = r.get(opened_at_key(circuit.name))
|
|
246
|
+
futures[:reserved_at] = r.get(reserved_at_key(circuit.name))
|
|
231
247
|
futures[:entries] = r.lrange(entries_key(circuit.name), 0, -1)
|
|
232
248
|
end
|
|
233
249
|
|
|
234
|
-
|
|
235
|
-
opened_at = futures[:opened_at].value ? Float(futures[:opened_at].value) : nil
|
|
236
|
-
opened_at = Faulty.current_time - options.circuit_ttl if state == :open && opened_at.nil?
|
|
237
|
-
|
|
238
|
-
Faulty::Status.from_entries(
|
|
239
|
-
map_entries(futures[:entries].value),
|
|
240
|
-
state: state,
|
|
241
|
-
lock: futures[:lock].value&.to_sym,
|
|
242
|
-
opened_at: opened_at,
|
|
243
|
-
options: circuit.options
|
|
244
|
-
)
|
|
250
|
+
build_status(circuit, futures)
|
|
245
251
|
end
|
|
246
252
|
|
|
247
253
|
# Get the circuit history up to `max_sample_size`
|
|
@@ -285,6 +291,26 @@ class Faulty
|
|
|
285
291
|
|
|
286
292
|
private
|
|
287
293
|
|
|
294
|
+
# Build a {Status} from the redis values fetched by {#status}
|
|
295
|
+
def build_status(circuit, futures)
|
|
296
|
+
state = futures[:state].value&.to_sym || :closed
|
|
297
|
+
opened_at = parse_float(futures[:opened_at].value)
|
|
298
|
+
opened_at = Faulty.current_time - options.circuit_ttl if state == :open && opened_at.nil?
|
|
299
|
+
|
|
300
|
+
Faulty::Status.from_entries(
|
|
301
|
+
map_entries(futures[:entries].value),
|
|
302
|
+
state: state,
|
|
303
|
+
lock: futures[:lock].value&.to_sym,
|
|
304
|
+
opened_at: opened_at,
|
|
305
|
+
reserved_at: parse_float(futures[:reserved_at].value),
|
|
306
|
+
options: circuit.options
|
|
307
|
+
)
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
def parse_float(value)
|
|
311
|
+
value ? Float(value) : nil
|
|
312
|
+
end
|
|
313
|
+
|
|
288
314
|
# Generate a key from its parts
|
|
289
315
|
#
|
|
290
316
|
# @return [String] The key
|
|
@@ -321,6 +347,11 @@ class Faulty
|
|
|
321
347
|
ckey(circuit_name, 'opened_at')
|
|
322
348
|
end
|
|
323
349
|
|
|
350
|
+
# @return [String] The key for circuit reserved_at
|
|
351
|
+
def reserved_at_key(circuit_name)
|
|
352
|
+
ckey(circuit_name, 'reserved_at')
|
|
353
|
+
end
|
|
354
|
+
|
|
324
355
|
# Get the current key to add circuit names to
|
|
325
356
|
def list_key
|
|
326
357
|
key('list', current_list_block)
|
|
@@ -387,9 +418,9 @@ class Faulty
|
|
|
387
418
|
#
|
|
388
419
|
# @yield [Redis] Yields the connection to the block
|
|
389
420
|
# @return The value returned from the block
|
|
390
|
-
def redis(&
|
|
421
|
+
def redis(&)
|
|
391
422
|
if options.client.respond_to?(:with)
|
|
392
|
-
options.client.with(&
|
|
423
|
+
options.client.with(&)
|
|
393
424
|
else
|
|
394
425
|
yield options.client
|
|
395
426
|
end
|
data/lib/faulty/version.rb
CHANGED
data/lib/faulty.rb
CHANGED
|
@@ -41,12 +41,12 @@ class Faulty
|
|
|
41
41
|
# @param config [Hash] Attributes for {Faulty::Options}
|
|
42
42
|
# @yield [Faulty::Options] For setting options in a block
|
|
43
43
|
# @return [self]
|
|
44
|
-
def init(default_name = :default, **config, &
|
|
44
|
+
def init(default_name = :default, **config, &)
|
|
45
45
|
raise AlreadyInitializedError if @instances
|
|
46
46
|
|
|
47
47
|
@default_instance = default_name
|
|
48
48
|
@instances = Concurrent::Map.new
|
|
49
|
-
register(default_name, new(**config, &
|
|
49
|
+
register(default_name, new(**config, &)) unless default_name.nil?
|
|
50
50
|
self
|
|
51
51
|
rescue StandardError
|
|
52
52
|
@instances = nil
|
|
@@ -110,8 +110,8 @@ class Faulty
|
|
|
110
110
|
# @param (see Faulty#circuit)
|
|
111
111
|
# @yield (see Faulty#circuit)
|
|
112
112
|
# @return (see Faulty#circuit)
|
|
113
|
-
def circuit(name, **config, &
|
|
114
|
-
default.circuit(name, **config, &
|
|
113
|
+
def circuit(name, **config, &)
|
|
114
|
+
default.circuit(name, **config, &)
|
|
115
115
|
end
|
|
116
116
|
|
|
117
117
|
# Get a list of all circuit names for the default instance
|
|
@@ -125,9 +125,11 @@ class Faulty
|
|
|
125
125
|
# The current time
|
|
126
126
|
#
|
|
127
127
|
# Used by Faulty wherever the current time is needed. Can be overridden
|
|
128
|
-
# for testing
|
|
128
|
+
# for testing. Returned as a Float (Unix epoch seconds with sub-second
|
|
129
|
+
# precision) so it can be stored in numeric Redis fields and compared
|
|
130
|
+
# against other timestamps without conversion.
|
|
129
131
|
#
|
|
130
|
-
# @return [
|
|
132
|
+
# @return [Float] The current time as a Unix timestamp
|
|
131
133
|
def current_time
|
|
132
134
|
Time.now.to_f
|
|
133
135
|
end
|
|
@@ -236,8 +238,8 @@ class Faulty
|
|
|
236
238
|
# @see Options
|
|
237
239
|
# @param options [Hash] Attributes for {Options}
|
|
238
240
|
# @yield [Options] For setting options in a block
|
|
239
|
-
def initialize(**options, &
|
|
240
|
-
@options = Options.new(options, &
|
|
241
|
+
def initialize(**options, &)
|
|
242
|
+
@options = Options.new(options, &)
|
|
241
243
|
@registry = CircuitRegistry.new(circuit_options)
|
|
242
244
|
end
|
|
243
245
|
|
|
@@ -252,9 +254,9 @@ class Faulty
|
|
|
252
254
|
# @param options [Hash] Attributes for {Circuit::Options}
|
|
253
255
|
# @yield [Circuit::Options] For setting options in a block
|
|
254
256
|
# @return [Circuit] The new circuit or the existing circuit if it already exists
|
|
255
|
-
def circuit(name, **options, &
|
|
257
|
+
def circuit(name, **options, &)
|
|
256
258
|
name = name.to_s
|
|
257
|
-
@registry.retrieve(name, options, &
|
|
259
|
+
@registry.retrieve(name, options, &)
|
|
258
260
|
end
|
|
259
261
|
|
|
260
262
|
# Get a list of all circuit names
|
|
@@ -285,7 +287,7 @@ class Faulty
|
|
|
285
287
|
# @return [Hash] The circuit options
|
|
286
288
|
def circuit_options
|
|
287
289
|
@options.to_h
|
|
288
|
-
.
|
|
290
|
+
.slice(:cache, :storage, :notifier)
|
|
289
291
|
.merge(options.circuit_defaults)
|
|
290
292
|
end
|
|
291
293
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: faulty
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.13.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Justin Howard
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-05-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|
|
@@ -94,7 +94,7 @@ dependencies:
|
|
|
94
94
|
- - ">="
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
96
|
version: '0.9'
|
|
97
|
-
description:
|
|
97
|
+
description:
|
|
98
98
|
email:
|
|
99
99
|
- jmhoward0@gmail.com
|
|
100
100
|
executables: []
|
|
@@ -152,8 +152,8 @@ licenses:
|
|
|
152
152
|
metadata:
|
|
153
153
|
rubygems_mfa_required: 'true'
|
|
154
154
|
changelog_uri: https://github.com/ParentSquare/faulty/blob/master/CHANGELOG.md
|
|
155
|
-
documentation_uri: https://www.rubydoc.info/gems/faulty/0.
|
|
156
|
-
post_install_message:
|
|
155
|
+
documentation_uri: https://www.rubydoc.info/gems/faulty/0.13.0
|
|
156
|
+
post_install_message:
|
|
157
157
|
rdoc_options: []
|
|
158
158
|
require_paths:
|
|
159
159
|
- lib
|
|
@@ -161,15 +161,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
161
161
|
requirements:
|
|
162
162
|
- - ">="
|
|
163
163
|
- !ruby/object:Gem::Version
|
|
164
|
-
version: '
|
|
164
|
+
version: '3.1'
|
|
165
165
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
166
166
|
requirements:
|
|
167
167
|
- - ">="
|
|
168
168
|
- !ruby/object:Gem::Version
|
|
169
169
|
version: '0'
|
|
170
170
|
requirements: []
|
|
171
|
-
rubygems_version: 3.
|
|
172
|
-
signing_key:
|
|
171
|
+
rubygems_version: 3.4.20
|
|
172
|
+
signing_key:
|
|
173
173
|
specification_version: 4
|
|
174
174
|
summary: Fault-tolerance tools for ruby based on circuit-breakers
|
|
175
175
|
test_files: []
|