faulty 0.1.4 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +49 -0
- data/.rubocop.yml +9 -0
- data/CHANGELOG.md +55 -0
- data/Gemfile +8 -3
- data/README.md +883 -310
- data/bin/check-version +5 -1
- data/faulty.gemspec +1 -1
- data/lib/faulty.rb +167 -43
- data/lib/faulty/cache.rb +3 -1
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +10 -21
- data/lib/faulty/cache/fault_tolerant_proxy.rb +15 -4
- data/lib/faulty/cache/interface.rb +1 -1
- data/lib/faulty/cache/mock.rb +1 -1
- data/lib/faulty/cache/null.rb +1 -1
- data/lib/faulty/cache/rails.rb +9 -10
- data/lib/faulty/circuit.rb +31 -16
- data/lib/faulty/error.rb +29 -7
- data/lib/faulty/events.rb +1 -1
- data/lib/faulty/events/callback_listener.rb +1 -1
- data/lib/faulty/events/honeybadger_listener.rb +1 -1
- data/lib/faulty/events/listener_interface.rb +1 -1
- data/lib/faulty/events/log_listener.rb +5 -6
- data/lib/faulty/events/notifier.rb +1 -1
- data/lib/faulty/immutable_options.rb +1 -1
- data/lib/faulty/patch.rb +154 -0
- data/lib/faulty/patch/base.rb +46 -0
- data/lib/faulty/patch/redis.rb +60 -0
- data/lib/faulty/result.rb +2 -2
- data/lib/faulty/status.rb +3 -2
- data/lib/faulty/storage.rb +4 -1
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +52 -57
- data/lib/faulty/storage/interface.rb +3 -2
- data/lib/faulty/storage/memory.rb +8 -4
- data/lib/faulty/storage/redis.rb +75 -13
- data/lib/faulty/version.rb +2 -2
- metadata +14 -7
- data/.travis.yml +0 -46
- data/lib/faulty/scope.rb +0 -117
@@ -1,15 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
5
|
# A wrapper for storage backends that may raise errors
|
6
6
|
#
|
7
|
-
# {
|
7
|
+
# {Faulty#initialize} automatically wraps all non-fault-tolerant storage backends with
|
8
8
|
# this class.
|
9
9
|
#
|
10
10
|
# If the storage backend raises a `StandardError`, it will be captured and
|
11
11
|
# sent to the notifier.
|
12
12
|
class FaultTolerantProxy
|
13
|
+
extend Forwardable
|
14
|
+
|
13
15
|
attr_reader :options
|
14
16
|
|
15
17
|
# Options for {FaultTolerantProxy}
|
@@ -36,6 +38,53 @@ module Faulty
|
|
36
38
|
@options = Options.new(options, &block)
|
37
39
|
end
|
38
40
|
|
41
|
+
# Wrap a storage backend in a FaultTolerantProxy unless it's already
|
42
|
+
# fault tolerant
|
43
|
+
#
|
44
|
+
# @param storage [Storage::Interface] The storage to maybe wrap
|
45
|
+
# @return [Storage::Interface] The original storage or a {FaultTolerantProxy}
|
46
|
+
def self.wrap(storage, **options, &block)
|
47
|
+
return storage if storage.fault_tolerant?
|
48
|
+
|
49
|
+
new(storage, **options, &block)
|
50
|
+
end
|
51
|
+
|
52
|
+
# @!method lock(circuit, state)
|
53
|
+
# Lock is not called in normal operation, so it doesn't capture errors
|
54
|
+
#
|
55
|
+
# @see Interface#lock
|
56
|
+
# @param (see Interface#lock)
|
57
|
+
# @return (see Interface#lock)
|
58
|
+
#
|
59
|
+
# @!method unlock(circuit)
|
60
|
+
# Unlock is not called in normal operation, so it doesn't capture errors
|
61
|
+
#
|
62
|
+
# @see Interface#unlock
|
63
|
+
# @param (see Interface#unlock)
|
64
|
+
# @return (see Interface#unlock)
|
65
|
+
#
|
66
|
+
# @!method reset(circuit)
|
67
|
+
# Reset is not called in normal operation, so it doesn't capture errors
|
68
|
+
#
|
69
|
+
# @see Interface#reset
|
70
|
+
# @param (see Interface#reset)
|
71
|
+
# @return (see Interface#reset)
|
72
|
+
#
|
73
|
+
# @!method history(circuit)
|
74
|
+
# History is not called in normal operation, so it doesn't capture errors
|
75
|
+
#
|
76
|
+
# @see Interface#history
|
77
|
+
# @param (see Interface#history)
|
78
|
+
# @return (see Interface#history)
|
79
|
+
#
|
80
|
+
# @!method list
|
81
|
+
# List is not called in normal operation, so it doesn't capture errors
|
82
|
+
#
|
83
|
+
# @see Interface#list
|
84
|
+
# @param (see Interface#list)
|
85
|
+
# @return (see Interface#list)
|
86
|
+
def_delegators :@storage, :lock, :unlock, :reset, :history, :list
|
87
|
+
|
39
88
|
# Add a history entry safely
|
40
89
|
#
|
41
90
|
# @see Interface#entry
|
@@ -45,7 +94,7 @@ module Faulty
|
|
45
94
|
@storage.entry(circuit, time, success)
|
46
95
|
rescue StandardError => e
|
47
96
|
options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e)
|
48
|
-
|
97
|
+
[]
|
49
98
|
end
|
50
99
|
|
51
100
|
# Safely mark a circuit as open
|
@@ -84,36 +133,6 @@ module Faulty
|
|
84
133
|
false
|
85
134
|
end
|
86
135
|
|
87
|
-
# Since lock is not called in normal operation, it does not capture
|
88
|
-
# errors
|
89
|
-
#
|
90
|
-
# @see Interface#lock
|
91
|
-
# @param (see Interface#lock)
|
92
|
-
# @return (see Interface#lock)
|
93
|
-
def lock(circuit, state)
|
94
|
-
@storage.lock(circuit, state)
|
95
|
-
end
|
96
|
-
|
97
|
-
# Since unlock is not called in normal operation, it does not capture
|
98
|
-
# errors
|
99
|
-
#
|
100
|
-
# @see Interface#unlock
|
101
|
-
# @param (see Interface#unlock)
|
102
|
-
# @return (see Interface#unlock)
|
103
|
-
def unlock(circuit)
|
104
|
-
@storage.unlock(circuit)
|
105
|
-
end
|
106
|
-
|
107
|
-
# Since reset is not called in normal operation, it does not capture
|
108
|
-
# errors
|
109
|
-
#
|
110
|
-
# @see Interface#reset
|
111
|
-
# @param (see Interface#reset)
|
112
|
-
# @return (see Interface#reset)
|
113
|
-
def reset(circuit)
|
114
|
-
@storage.reset(circuit)
|
115
|
-
end
|
116
|
-
|
117
136
|
# Safely get the status of a circuit
|
118
137
|
#
|
119
138
|
# If the backend is unavailable, this returns a stub status that
|
@@ -129,30 +148,6 @@ module Faulty
|
|
129
148
|
stub_status(circuit)
|
130
149
|
end
|
131
150
|
|
132
|
-
# Since history is not called in normal operation, it does not capture
|
133
|
-
# errors
|
134
|
-
#
|
135
|
-
# @see Interface#history
|
136
|
-
# @param (see Interface#history)
|
137
|
-
# @return (see Interface#history)
|
138
|
-
def history(circuit)
|
139
|
-
@storage.history(circuit)
|
140
|
-
end
|
141
|
-
|
142
|
-
# Safely get the list of circuit names
|
143
|
-
#
|
144
|
-
# If the backend is unavailable, this returns an empty array
|
145
|
-
#
|
146
|
-
# @see Interface#list
|
147
|
-
# @param (see Interface#list)
|
148
|
-
# @return (see Interface#list)
|
149
|
-
def list
|
150
|
-
@storage.list
|
151
|
-
rescue StandardError => e
|
152
|
-
options.notifier.notify(:storage_failure, action: :list, error: e)
|
153
|
-
[]
|
154
|
-
end
|
155
|
-
|
156
151
|
# This cache makes any storage fault tolerant, so this is always `true`
|
157
152
|
#
|
158
153
|
# @return [true]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
5
|
# The interface required for a storage backend implementation
|
6
6
|
#
|
@@ -14,7 +14,8 @@ module Faulty
|
|
14
14
|
# @param circuit [Circuit] The circuit that ran
|
15
15
|
# @param time [Integer] The unix timestamp for the run
|
16
16
|
# @param success [Boolean] True if the run succeeded
|
17
|
-
# @return [
|
17
|
+
# @return [Array<Array>] An array of the new history tuples after adding
|
18
|
+
# the new entry, see {#history}
|
18
19
|
def entry(circuit, time, success)
|
19
20
|
raise NotImplementedError
|
20
21
|
end
|
@@ -1,11 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
5
|
# The default in-memory storage for circuits
|
6
6
|
#
|
7
|
-
# This implementation is
|
8
|
-
#
|
7
|
+
# This implementation is thread-safe and circuit state is shared across
|
8
|
+
# threads. Since state is stored in-memory, this state is not shared across
|
9
|
+
# processes, or persisted across application restarts.
|
9
10
|
#
|
10
11
|
# Circuit state and runs are stored in memory. Although runs have a maximum
|
11
12
|
# size within a circuit, there is no limit on the number of circuits that
|
@@ -18,6 +19,9 @@ module Faulty
|
|
18
19
|
#
|
19
20
|
# This can be used as a reference implementation for storage backends that
|
20
21
|
# store a list of circuit run entries.
|
22
|
+
#
|
23
|
+
# @todo Add a more sophsticated implmentation that can limit the number of
|
24
|
+
# circuits stored.
|
21
25
|
class Memory
|
22
26
|
attr_reader :options
|
23
27
|
|
@@ -85,7 +89,7 @@ module Faulty
|
|
85
89
|
runs.push([time, success])
|
86
90
|
runs.shift if runs.size > options.max_sample_size
|
87
91
|
end
|
88
|
-
memory.
|
92
|
+
memory.runs.value
|
89
93
|
end
|
90
94
|
|
91
95
|
# Mark a circuit as open
|
data/lib/faulty/storage/redis.rb
CHANGED
@@ -1,7 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
|
+
# A storage backend for storing circuit state in Redis.
|
6
|
+
#
|
7
|
+
# When using this or any networked backend, be sure to evaluate the risk,
|
8
|
+
# and set conservative timeouts so that the circuit storage does not cause
|
9
|
+
# cascading failures in your application when evaluating circuits. Always
|
10
|
+
# wrap this backend with a {FaultTolerantProxy} to limit the effect of
|
11
|
+
# these types of events.
|
5
12
|
class Redis # rubocop:disable Metrics/ClassLength
|
6
13
|
# Separates the time/status for history entry strings
|
7
14
|
ENTRY_SEPARATOR = ':'
|
@@ -27,12 +34,17 @@ module Faulty
|
|
27
34
|
# circuit run history entry. Default `100`.
|
28
35
|
# @!attribute [r] circuit_ttl
|
29
36
|
# @return [Integer] The maximum number of seconds to keep a circuit.
|
30
|
-
# A value of `nil` disables circuit expiration.
|
37
|
+
# A value of `nil` disables circuit expiration. This does not apply to
|
38
|
+
# locks, which have an indefinite storage time.
|
31
39
|
# Default `604_800` (1 week).
|
32
40
|
# @!attribute [r] list_granularity
|
33
41
|
# @return [Integer] The number of seconds after which a new set is
|
34
42
|
# created to store circuit names. The old set is kept until
|
35
43
|
# circuit_ttl expires. Default `3600` (1 hour).
|
44
|
+
# @!attribute [r] disable_warnings
|
45
|
+
# @return [Boolean] By default, this class warns if the client options
|
46
|
+
# are outside the recommended values. Set to true to disable these
|
47
|
+
# warnings.
|
36
48
|
Options = Struct.new(
|
37
49
|
:client,
|
38
50
|
:key_prefix,
|
@@ -40,7 +52,8 @@ module Faulty
|
|
40
52
|
:max_sample_size,
|
41
53
|
:sample_ttl,
|
42
54
|
:circuit_ttl,
|
43
|
-
:list_granularity
|
55
|
+
:list_granularity,
|
56
|
+
:disable_warnings
|
44
57
|
) do
|
45
58
|
include ImmutableOptions
|
46
59
|
|
@@ -53,7 +66,8 @@ module Faulty
|
|
53
66
|
max_sample_size: 100,
|
54
67
|
sample_ttl: 1800,
|
55
68
|
circuit_ttl: 604_800,
|
56
|
-
list_granularity: 3600
|
69
|
+
list_granularity: 3600,
|
70
|
+
disable_warnings: false
|
57
71
|
}
|
58
72
|
end
|
59
73
|
|
@@ -62,7 +76,7 @@ module Faulty
|
|
62
76
|
end
|
63
77
|
|
64
78
|
def finalize
|
65
|
-
self.client = ::Redis.new unless client
|
79
|
+
self.client = ::Redis.new(timeout: 1) unless client
|
66
80
|
end
|
67
81
|
end
|
68
82
|
|
@@ -70,6 +84,8 @@ module Faulty
|
|
70
84
|
# @yield [Options] For setting options in a block
|
71
85
|
def initialize(**options, &block)
|
72
86
|
@options = Options.new(options, &block)
|
87
|
+
|
88
|
+
check_client_options!
|
73
89
|
end
|
74
90
|
|
75
91
|
# Add an entry to storage
|
@@ -79,15 +95,15 @@ module Faulty
|
|
79
95
|
# @return (see Interface#entry)
|
80
96
|
def entry(circuit, time, success)
|
81
97
|
key = entries_key(circuit)
|
82
|
-
pipe do |r|
|
98
|
+
result = pipe do |r|
|
83
99
|
r.sadd(list_key, circuit.name)
|
84
100
|
r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
|
85
101
|
r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
|
86
102
|
r.ltrim(key, 0, options.max_sample_size - 1)
|
87
103
|
r.expire(key, options.sample_ttl) if options.sample_ttl
|
104
|
+
r.lrange(key, 0, -1)
|
88
105
|
end
|
89
|
-
|
90
|
-
status(circuit)
|
106
|
+
map_entries(result.last)
|
91
107
|
end
|
92
108
|
|
93
109
|
# Mark a circuit as open
|
@@ -97,7 +113,7 @@ module Faulty
|
|
97
113
|
# @return (see Interface#open)
|
98
114
|
def open(circuit, opened_at)
|
99
115
|
redis do |r|
|
100
|
-
opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
|
116
|
+
opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open', ex: options.circuit_ttl)
|
101
117
|
r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
|
102
118
|
opened
|
103
119
|
end
|
@@ -110,7 +126,7 @@ module Faulty
|
|
110
126
|
# @return (see Interface#reopen)
|
111
127
|
def reopen(circuit, opened_at, previous_opened_at)
|
112
128
|
redis do |r|
|
113
|
-
compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
|
129
|
+
compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at, ex: options.circuit_ttl)
|
114
130
|
end
|
115
131
|
end
|
116
132
|
|
@@ -121,7 +137,7 @@ module Faulty
|
|
121
137
|
# @return (see Interface#close)
|
122
138
|
def close(circuit)
|
123
139
|
redis do |r|
|
124
|
-
closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
|
140
|
+
closed = compare_and_set(r, state_key(circuit), ['open'], 'closed', ex: options.circuit_ttl)
|
125
141
|
r.del(entries_key(circuit)) if closed
|
126
142
|
closed
|
127
143
|
end
|
@@ -196,6 +212,9 @@ module Faulty
|
|
196
212
|
map_entries(entries).reverse
|
197
213
|
end
|
198
214
|
|
215
|
+
# List all unexpired circuits
|
216
|
+
#
|
217
|
+
# @return (see Interface#list)
|
199
218
|
def list
|
200
219
|
redis { |r| r.sunion(*all_list_keys) }
|
201
220
|
end
|
@@ -287,10 +306,10 @@ module Faulty
|
|
287
306
|
# @param new [String] The new value to set if the compare passes
|
288
307
|
# @return [Boolean] True if the value was set to `new`, false if the CAS
|
289
308
|
# failed
|
290
|
-
def compare_and_set(redis, key, old, new)
|
309
|
+
def compare_and_set(redis, key, old, new, ex:)
|
291
310
|
redis.watch(key) do
|
292
311
|
if old.include?(redis.get(key))
|
293
|
-
result = redis.multi { |m| m.set(key, new) }
|
312
|
+
result = redis.multi { |m| m.set(key, new, ex: ex) }
|
294
313
|
result && result[0] == 'OK'
|
295
314
|
else
|
296
315
|
redis.unwatch
|
@@ -330,6 +349,49 @@ module Faulty
|
|
330
349
|
[time.to_i, state == '1']
|
331
350
|
end
|
332
351
|
end
|
352
|
+
|
353
|
+
def check_client_options!
|
354
|
+
return if options.disable_warnings
|
355
|
+
|
356
|
+
check_redis_options!
|
357
|
+
check_pool_options!
|
358
|
+
rescue StandardError => e
|
359
|
+
warn "Faulty error while checking client options: #{e.message}"
|
360
|
+
end
|
361
|
+
|
362
|
+
def check_redis_options!
|
363
|
+
ropts = redis { |r| r.instance_variable_get(:@client).options }
|
364
|
+
|
365
|
+
bad_timeouts = {}
|
366
|
+
%i[connect_timeout read_timeout write_timeout].each do |time_opt|
|
367
|
+
bad_timeouts[time_opt] = ropts[time_opt] if ropts[time_opt] > 2
|
368
|
+
end
|
369
|
+
|
370
|
+
unless bad_timeouts.empty?
|
371
|
+
warn <<~MSG
|
372
|
+
Faulty recommends setting Redis timeouts <= 2 to prevent cascading
|
373
|
+
failures when evaluating circuits. Your options are:
|
374
|
+
#{bad_timeouts}
|
375
|
+
MSG
|
376
|
+
end
|
377
|
+
|
378
|
+
if ropts[:reconnect_attempts] > 1
|
379
|
+
warn <<~MSG
|
380
|
+
Faulty recommends setting Redis reconnect_attempts to <= 1 to
|
381
|
+
prevent cascading failures. Your setting is #{ropts[:reconnect_attempts]}
|
382
|
+
MSG
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
def check_pool_options!
|
387
|
+
if options.client.class.name == 'ConnectionPool'
|
388
|
+
timeout = options.client.instance_variable_get(:@timeout)
|
389
|
+
warn(<<~MSG) if timeout > 2
|
390
|
+
Faulty recommends setting ConnectionPool timeouts <= 2 to prevent
|
391
|
+
cascading failures when evaluating circuits. Your setting is #{timeout}
|
392
|
+
MSG
|
393
|
+
end
|
394
|
+
end
|
333
395
|
end
|
334
396
|
end
|
335
397
|
end
|
data/lib/faulty/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: faulty
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Howard
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -56,14 +56,14 @@ dependencies:
|
|
56
56
|
name: redis
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - ">="
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '3.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '3.0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
@@ -129,10 +129,10 @@ executables: []
|
|
129
129
|
extensions: []
|
130
130
|
extra_rdoc_files: []
|
131
131
|
files:
|
132
|
+
- ".github/workflows/ci.yml"
|
132
133
|
- ".gitignore"
|
133
134
|
- ".rspec"
|
134
135
|
- ".rubocop.yml"
|
135
|
-
- ".travis.yml"
|
136
136
|
- ".yardopts"
|
137
137
|
- CHANGELOG.md
|
138
138
|
- Gemfile
|
@@ -148,6 +148,8 @@ files:
|
|
148
148
|
- faulty.gemspec
|
149
149
|
- lib/faulty.rb
|
150
150
|
- lib/faulty/cache.rb
|
151
|
+
- lib/faulty/cache/auto_wire.rb
|
152
|
+
- lib/faulty/cache/circuit_proxy.rb
|
151
153
|
- lib/faulty/cache/default.rb
|
152
154
|
- lib/faulty/cache/fault_tolerant_proxy.rb
|
153
155
|
- lib/faulty/cache/interface.rb
|
@@ -163,10 +165,15 @@ files:
|
|
163
165
|
- lib/faulty/events/log_listener.rb
|
164
166
|
- lib/faulty/events/notifier.rb
|
165
167
|
- lib/faulty/immutable_options.rb
|
168
|
+
- lib/faulty/patch.rb
|
169
|
+
- lib/faulty/patch/base.rb
|
170
|
+
- lib/faulty/patch/redis.rb
|
166
171
|
- lib/faulty/result.rb
|
167
|
-
- lib/faulty/scope.rb
|
168
172
|
- lib/faulty/status.rb
|
169
173
|
- lib/faulty/storage.rb
|
174
|
+
- lib/faulty/storage/auto_wire.rb
|
175
|
+
- lib/faulty/storage/circuit_proxy.rb
|
176
|
+
- lib/faulty/storage/fallback_chain.rb
|
170
177
|
- lib/faulty/storage/fault_tolerant_proxy.rb
|
171
178
|
- lib/faulty/storage/interface.rb
|
172
179
|
- lib/faulty/storage/memory.rb
|
@@ -191,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
191
198
|
- !ruby/object:Gem::Version
|
192
199
|
version: '0'
|
193
200
|
requirements: []
|
194
|
-
rubygems_version: 3.
|
201
|
+
rubygems_version: 3.1.2
|
195
202
|
signing_key:
|
196
203
|
specification_version: 4
|
197
204
|
summary: Fault-tolerance tools for ruby based on circuit-breakers
|