faulty 0.1.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +49 -0
- data/.rubocop.yml +9 -0
- data/CHANGELOG.md +50 -2
- data/Gemfile +22 -0
- data/README.md +836 -220
- data/bin/check-version +5 -1
- data/bin/console +1 -1
- data/faulty.gemspec +4 -11
- data/lib/faulty.rb +157 -43
- data/lib/faulty/cache.rb +3 -1
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +10 -21
- data/lib/faulty/cache/fault_tolerant_proxy.rb +15 -4
- data/lib/faulty/cache/interface.rb +1 -1
- data/lib/faulty/cache/mock.rb +1 -1
- data/lib/faulty/cache/null.rb +1 -1
- data/lib/faulty/cache/rails.rb +9 -10
- data/lib/faulty/circuit.rb +10 -5
- data/lib/faulty/error.rb +18 -4
- data/lib/faulty/events.rb +3 -2
- data/lib/faulty/events/callback_listener.rb +1 -1
- data/lib/faulty/events/honeybadger_listener.rb +53 -0
- data/lib/faulty/events/listener_interface.rb +1 -1
- data/lib/faulty/events/log_listener.rb +5 -6
- data/lib/faulty/events/notifier.rb +11 -2
- data/lib/faulty/immutable_options.rb +1 -1
- data/lib/faulty/result.rb +2 -2
- data/lib/faulty/status.rb +3 -2
- data/lib/faulty/storage.rb +4 -1
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +51 -56
- data/lib/faulty/storage/interface.rb +1 -1
- data/lib/faulty/storage/memory.rb +8 -4
- data/lib/faulty/storage/redis.rb +75 -13
- data/lib/faulty/version.rb +2 -2
- metadata +18 -122
- data/.travis.yml +0 -44
- data/lib/faulty/scope.rb +0 -117
@@ -1,11 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
5
|
# The default in-memory storage for circuits
|
6
6
|
#
|
7
|
-
# This implementation is
|
8
|
-
#
|
7
|
+
# This implementation is thread-safe and circuit state is shared across
|
8
|
+
# threads. Since state is stored in-memory, this state is not shared across
|
9
|
+
# processes, or persisted across application restarts.
|
9
10
|
#
|
10
11
|
# Circuit state and runs are stored in memory. Although runs have a maximum
|
11
12
|
# size within a circuit, there is no limit on the number of circuits that
|
@@ -18,6 +19,9 @@ module Faulty
|
|
18
19
|
#
|
19
20
|
# This can be used as a reference implementation for storage backends that
|
20
21
|
# store a list of circuit run entries.
|
22
|
+
#
|
23
|
+
# @todo Add a more sophsticated implmentation that can limit the number of
|
24
|
+
# circuits stored.
|
21
25
|
class Memory
|
22
26
|
attr_reader :options
|
23
27
|
|
@@ -83,7 +87,7 @@ module Faulty
|
|
83
87
|
memory = fetch(circuit)
|
84
88
|
memory.runs.borrow do |runs|
|
85
89
|
runs.push([time, success])
|
86
|
-
runs.
|
90
|
+
runs.shift if runs.size > options.max_sample_size
|
87
91
|
end
|
88
92
|
memory.status(circuit.options)
|
89
93
|
end
|
data/lib/faulty/storage/redis.rb
CHANGED
@@ -1,7 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Storage
|
5
|
+
# A storage backend for storing circuit state in Redis.
|
6
|
+
#
|
7
|
+
# When using this or any networked backend, be sure to evaluate the risk,
|
8
|
+
# and set conservative timeouts so that the circuit storage does not cause
|
9
|
+
# cascading failures in your application when evaluating circuits. Always
|
10
|
+
# wrap this backend with a {FaultTolerantProxy} to limit the effect of
|
11
|
+
# these types of events.
|
5
12
|
class Redis # rubocop:disable Metrics/ClassLength
|
6
13
|
# Separates the time/status for history entry strings
|
7
14
|
ENTRY_SEPARATOR = ':'
|
@@ -27,12 +34,17 @@ module Faulty
|
|
27
34
|
# circuit run history entry. Default `100`.
|
28
35
|
# @!attribute [r] circuit_ttl
|
29
36
|
# @return [Integer] The maximum number of seconds to keep a circuit.
|
30
|
-
# A value of `nil` disables circuit expiration.
|
37
|
+
# A value of `nil` disables circuit expiration. This does not apply to
|
38
|
+
# locks, which have an indefinite storage time.
|
31
39
|
# Default `604_800` (1 week).
|
32
40
|
# @!attribute [r] list_granularity
|
33
41
|
# @return [Integer] The number of seconds after which a new set is
|
34
42
|
# created to store circuit names. The old set is kept until
|
35
43
|
# circuit_ttl expires. Default `3600` (1 hour).
|
44
|
+
# @!attribute [r] disable_warnings
|
45
|
+
# @return [Boolean] By default, this class warns if the client options
|
46
|
+
# are outside the recommended values. Set to true to disable these
|
47
|
+
# warnings.
|
36
48
|
Options = Struct.new(
|
37
49
|
:client,
|
38
50
|
:key_prefix,
|
@@ -40,7 +52,8 @@ module Faulty
|
|
40
52
|
:max_sample_size,
|
41
53
|
:sample_ttl,
|
42
54
|
:circuit_ttl,
|
43
|
-
:list_granularity
|
55
|
+
:list_granularity,
|
56
|
+
:disable_warnings
|
44
57
|
) do
|
45
58
|
include ImmutableOptions
|
46
59
|
|
@@ -53,7 +66,8 @@ module Faulty
|
|
53
66
|
max_sample_size: 100,
|
54
67
|
sample_ttl: 1800,
|
55
68
|
circuit_ttl: 604_800,
|
56
|
-
list_granularity: 3600
|
69
|
+
list_granularity: 3600,
|
70
|
+
disable_warnings: false
|
57
71
|
}
|
58
72
|
end
|
59
73
|
|
@@ -62,7 +76,7 @@ module Faulty
|
|
62
76
|
end
|
63
77
|
|
64
78
|
def finalize
|
65
|
-
self.client = ::Redis.new unless client
|
79
|
+
self.client = ::Redis.new(timeout: 1) unless client
|
66
80
|
end
|
67
81
|
end
|
68
82
|
|
@@ -70,6 +84,8 @@ module Faulty
|
|
70
84
|
# @yield [Options] For setting options in a block
|
71
85
|
def initialize(**options, &block)
|
72
86
|
@options = Options.new(options, &block)
|
87
|
+
|
88
|
+
check_client_options!
|
73
89
|
end
|
74
90
|
|
75
91
|
# Add an entry to storage
|
@@ -97,7 +113,7 @@ module Faulty
|
|
97
113
|
# @return (see Interface#open)
|
98
114
|
def open(circuit, opened_at)
|
99
115
|
redis do |r|
|
100
|
-
opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
|
116
|
+
opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open', ex: options.circuit_ttl)
|
101
117
|
r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
|
102
118
|
opened
|
103
119
|
end
|
@@ -110,7 +126,7 @@ module Faulty
|
|
110
126
|
# @return (see Interface#reopen)
|
111
127
|
def reopen(circuit, opened_at, previous_opened_at)
|
112
128
|
redis do |r|
|
113
|
-
compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
|
129
|
+
compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at, ex: options.circuit_ttl)
|
114
130
|
end
|
115
131
|
end
|
116
132
|
|
@@ -121,7 +137,7 @@ module Faulty
|
|
121
137
|
# @return (see Interface#close)
|
122
138
|
def close(circuit)
|
123
139
|
redis do |r|
|
124
|
-
closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
|
140
|
+
closed = compare_and_set(r, state_key(circuit), ['open'], 'closed', ex: options.circuit_ttl)
|
125
141
|
r.del(entries_key(circuit)) if closed
|
126
142
|
closed
|
127
143
|
end
|
@@ -196,6 +212,9 @@ module Faulty
|
|
196
212
|
map_entries(entries).reverse
|
197
213
|
end
|
198
214
|
|
215
|
+
# List all unexpired circuits
|
216
|
+
#
|
217
|
+
# @return (see Interface#list)
|
199
218
|
def list
|
200
219
|
redis { |r| r.sunion(*all_list_keys) }
|
201
220
|
end
|
@@ -287,16 +306,16 @@ module Faulty
|
|
287
306
|
# @param new [String] The new value to set if the compare passes
|
288
307
|
# @return [Boolean] True if the value was set to `new`, false if the CAS
|
289
308
|
# failed
|
290
|
-
def compare_and_set(redis, key, old, new)
|
291
|
-
|
309
|
+
def compare_and_set(redis, key, old, new, ex:)
|
310
|
+
redis.watch(key) do
|
292
311
|
if old.include?(redis.get(key))
|
293
|
-
redis.multi { |m| m.set(key, new) }
|
312
|
+
result = redis.multi { |m| m.set(key, new, ex: ex) }
|
313
|
+
result && result[0] == 'OK'
|
294
314
|
else
|
295
315
|
redis.unwatch
|
316
|
+
false
|
296
317
|
end
|
297
318
|
end
|
298
|
-
|
299
|
-
result[0] == 'OK'
|
300
319
|
end
|
301
320
|
|
302
321
|
# Yield a Redis connection
|
@@ -330,6 +349,49 @@ module Faulty
|
|
330
349
|
[time.to_i, state == '1']
|
331
350
|
end
|
332
351
|
end
|
352
|
+
|
353
|
+
def check_client_options!
|
354
|
+
return if options.disable_warnings
|
355
|
+
|
356
|
+
check_redis_options!
|
357
|
+
check_pool_options!
|
358
|
+
rescue StandardError => e
|
359
|
+
warn "Faulty error while checking client options: #{e.message}"
|
360
|
+
end
|
361
|
+
|
362
|
+
def check_redis_options!
|
363
|
+
ropts = redis { |r| r.instance_variable_get(:@client).options }
|
364
|
+
|
365
|
+
bad_timeouts = {}
|
366
|
+
%i[connect_timeout read_timeout write_timeout].each do |time_opt|
|
367
|
+
bad_timeouts[time_opt] = ropts[time_opt] if ropts[time_opt] > 2
|
368
|
+
end
|
369
|
+
|
370
|
+
unless bad_timeouts.empty?
|
371
|
+
warn <<~MSG
|
372
|
+
Faulty recommends setting Redis timeouts <= 2 to prevent cascading
|
373
|
+
failures when evaluating circuits. Your options are:
|
374
|
+
#{bad_timeouts}
|
375
|
+
MSG
|
376
|
+
end
|
377
|
+
|
378
|
+
if ropts[:reconnect_attempts] > 1
|
379
|
+
warn <<~MSG
|
380
|
+
Faulty recommends setting Redis reconnect_attempts to <= 1 to
|
381
|
+
prevent cascading failures. Your setting is #{ropts[:reconnect_attempts]}
|
382
|
+
MSG
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
def check_pool_options!
|
387
|
+
if options.client.class.name == 'ConnectionPool'
|
388
|
+
timeout = options.client.instance_variable_get(:@timeout)
|
389
|
+
warn(<<~MSG) if timeout > 2
|
390
|
+
Faulty recommends setting ConnectionPool timeouts <= 2 to prevent
|
391
|
+
cascading failures when evaluating circuits. Your setting is #{timeout}
|
392
|
+
MSG
|
393
|
+
end
|
394
|
+
end
|
333
395
|
end
|
334
396
|
end
|
335
397
|
end
|
data/lib/faulty/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: faulty
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Howard
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-02-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -24,54 +24,6 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: activesupport
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '4.2'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '4.2'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: bundler
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '1.17'
|
48
|
-
- - "<"
|
49
|
-
- !ruby/object:Gem::Version
|
50
|
-
version: '3'
|
51
|
-
type: :development
|
52
|
-
prerelease: false
|
53
|
-
version_requirements: !ruby/object:Gem::Requirement
|
54
|
-
requirements:
|
55
|
-
- - ">="
|
56
|
-
- !ruby/object:Gem::Version
|
57
|
-
version: '1.17'
|
58
|
-
- - "<"
|
59
|
-
- !ruby/object:Gem::Version
|
60
|
-
version: '3'
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: byebug
|
63
|
-
requirement: !ruby/object:Gem::Requirement
|
64
|
-
requirements:
|
65
|
-
- - "~>"
|
66
|
-
- !ruby/object:Gem::Version
|
67
|
-
version: '11.0'
|
68
|
-
type: :development
|
69
|
-
prerelease: false
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - "~>"
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '11.0'
|
75
27
|
- !ruby/object:Gem::Dependency
|
76
28
|
name: connection_pool
|
77
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -87,45 +39,31 @@ dependencies:
|
|
87
39
|
- !ruby/object:Gem::Version
|
88
40
|
version: '2.0'
|
89
41
|
- !ruby/object:Gem::Dependency
|
90
|
-
name:
|
42
|
+
name: honeybadger
|
91
43
|
requirement: !ruby/object:Gem::Requirement
|
92
44
|
requirements:
|
93
|
-
- - "
|
94
|
-
- !ruby/object:Gem::Version
|
95
|
-
version: '1.0'
|
96
|
-
type: :development
|
97
|
-
prerelease: false
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - "~>"
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: '1.0'
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
|
-
name: redcarpet
|
105
|
-
requirement: !ruby/object:Gem::Requirement
|
106
|
-
requirements:
|
107
|
-
- - "~>"
|
45
|
+
- - ">="
|
108
46
|
- !ruby/object:Gem::Version
|
109
|
-
version: '
|
47
|
+
version: '2.0'
|
110
48
|
type: :development
|
111
49
|
prerelease: false
|
112
50
|
version_requirements: !ruby/object:Gem::Requirement
|
113
51
|
requirements:
|
114
|
-
- - "
|
52
|
+
- - ">="
|
115
53
|
- !ruby/object:Gem::Version
|
116
|
-
version: '
|
54
|
+
version: '2.0'
|
117
55
|
- !ruby/object:Gem::Dependency
|
118
56
|
name: redis
|
119
57
|
requirement: !ruby/object:Gem::Requirement
|
120
58
|
requirements:
|
121
|
-
- - "
|
59
|
+
- - ">="
|
122
60
|
- !ruby/object:Gem::Version
|
123
61
|
version: '3.0'
|
124
62
|
type: :development
|
125
63
|
prerelease: false
|
126
64
|
version_requirements: !ruby/object:Gem::Requirement
|
127
65
|
requirements:
|
128
|
-
- - "
|
66
|
+
- - ">="
|
129
67
|
- !ruby/object:Gem::Version
|
130
68
|
version: '3.0'
|
131
69
|
- !ruby/object:Gem::Dependency
|
@@ -142,20 +80,6 @@ dependencies:
|
|
142
80
|
- - "~>"
|
143
81
|
- !ruby/object:Gem::Version
|
144
82
|
version: '3.8'
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
|
-
name: rspec_junit_formatter
|
147
|
-
requirement: !ruby/object:Gem::Requirement
|
148
|
-
requirements:
|
149
|
-
- - "~>"
|
150
|
-
- !ruby/object:Gem::Version
|
151
|
-
version: '0.4'
|
152
|
-
type: :development
|
153
|
-
prerelease: false
|
154
|
-
version_requirements: !ruby/object:Gem::Requirement
|
155
|
-
requirements:
|
156
|
-
- - "~>"
|
157
|
-
- !ruby/object:Gem::Version
|
158
|
-
version: '0.4'
|
159
83
|
- !ruby/object:Gem::Dependency
|
160
84
|
name: rubocop
|
161
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -184,26 +108,6 @@ dependencies:
|
|
184
108
|
- - '='
|
185
109
|
- !ruby/object:Gem::Version
|
186
110
|
version: 1.38.1
|
187
|
-
- !ruby/object:Gem::Dependency
|
188
|
-
name: simplecov
|
189
|
-
requirement: !ruby/object:Gem::Requirement
|
190
|
-
requirements:
|
191
|
-
- - ">="
|
192
|
-
- !ruby/object:Gem::Version
|
193
|
-
version: 0.17.1
|
194
|
-
- - "<"
|
195
|
-
- !ruby/object:Gem::Version
|
196
|
-
version: '0.18'
|
197
|
-
type: :development
|
198
|
-
prerelease: false
|
199
|
-
version_requirements: !ruby/object:Gem::Requirement
|
200
|
-
requirements:
|
201
|
-
- - ">="
|
202
|
-
- !ruby/object:Gem::Version
|
203
|
-
version: 0.17.1
|
204
|
-
- - "<"
|
205
|
-
- !ruby/object:Gem::Version
|
206
|
-
version: '0.18'
|
207
111
|
- !ruby/object:Gem::Dependency
|
208
112
|
name: timecop
|
209
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -218,20 +122,6 @@ dependencies:
|
|
218
122
|
- - ">="
|
219
123
|
- !ruby/object:Gem::Version
|
220
124
|
version: '0.9'
|
221
|
-
- !ruby/object:Gem::Dependency
|
222
|
-
name: yard
|
223
|
-
requirement: !ruby/object:Gem::Requirement
|
224
|
-
requirements:
|
225
|
-
- - "~>"
|
226
|
-
- !ruby/object:Gem::Version
|
227
|
-
version: 0.9.25
|
228
|
-
type: :development
|
229
|
-
prerelease: false
|
230
|
-
version_requirements: !ruby/object:Gem::Requirement
|
231
|
-
requirements:
|
232
|
-
- - "~>"
|
233
|
-
- !ruby/object:Gem::Version
|
234
|
-
version: 0.9.25
|
235
125
|
description:
|
236
126
|
email:
|
237
127
|
- jmhoward0@gmail.com
|
@@ -239,10 +129,10 @@ executables: []
|
|
239
129
|
extensions: []
|
240
130
|
extra_rdoc_files: []
|
241
131
|
files:
|
132
|
+
- ".github/workflows/ci.yml"
|
242
133
|
- ".gitignore"
|
243
134
|
- ".rspec"
|
244
135
|
- ".rubocop.yml"
|
245
|
-
- ".travis.yml"
|
246
136
|
- ".yardopts"
|
247
137
|
- CHANGELOG.md
|
248
138
|
- Gemfile
|
@@ -258,6 +148,8 @@ files:
|
|
258
148
|
- faulty.gemspec
|
259
149
|
- lib/faulty.rb
|
260
150
|
- lib/faulty/cache.rb
|
151
|
+
- lib/faulty/cache/auto_wire.rb
|
152
|
+
- lib/faulty/cache/circuit_proxy.rb
|
261
153
|
- lib/faulty/cache/default.rb
|
262
154
|
- lib/faulty/cache/fault_tolerant_proxy.rb
|
263
155
|
- lib/faulty/cache/interface.rb
|
@@ -268,14 +160,17 @@ files:
|
|
268
160
|
- lib/faulty/error.rb
|
269
161
|
- lib/faulty/events.rb
|
270
162
|
- lib/faulty/events/callback_listener.rb
|
163
|
+
- lib/faulty/events/honeybadger_listener.rb
|
271
164
|
- lib/faulty/events/listener_interface.rb
|
272
165
|
- lib/faulty/events/log_listener.rb
|
273
166
|
- lib/faulty/events/notifier.rb
|
274
167
|
- lib/faulty/immutable_options.rb
|
275
168
|
- lib/faulty/result.rb
|
276
|
-
- lib/faulty/scope.rb
|
277
169
|
- lib/faulty/status.rb
|
278
170
|
- lib/faulty/storage.rb
|
171
|
+
- lib/faulty/storage/auto_wire.rb
|
172
|
+
- lib/faulty/storage/circuit_proxy.rb
|
173
|
+
- lib/faulty/storage/fallback_chain.rb
|
279
174
|
- lib/faulty/storage/fault_tolerant_proxy.rb
|
280
175
|
- lib/faulty/storage/interface.rb
|
281
176
|
- lib/faulty/storage/memory.rb
|
@@ -300,7 +195,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
300
195
|
- !ruby/object:Gem::Version
|
301
196
|
version: '0'
|
302
197
|
requirements: []
|
303
|
-
|
198
|
+
rubyforge_project:
|
199
|
+
rubygems_version: 2.7.6
|
304
200
|
signing_key:
|
305
201
|
specification_version: 4
|
306
202
|
summary: Fault-tolerance tools for ruby based on circuit-breakers
|