faulty 0.2.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +56 -0
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +50 -0
- data/Gemfile +11 -3
- data/README.md +919 -303
- data/bin/check-version +5 -1
- data/faulty.gemspec +1 -2
- data/lib/faulty.rb +35 -23
- data/lib/faulty/cache.rb +2 -0
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +9 -20
- data/lib/faulty/cache/fault_tolerant_proxy.rb +13 -2
- data/lib/faulty/cache/rails.rb +8 -9
- data/lib/faulty/circuit.rb +30 -15
- data/lib/faulty/error.rb +25 -3
- data/lib/faulty/events/log_listener.rb +4 -5
- data/lib/faulty/patch.rb +154 -0
- data/lib/faulty/patch/base.rb +46 -0
- data/lib/faulty/patch/mysql2.rb +81 -0
- data/lib/faulty/patch/redis.rb +93 -0
- data/lib/faulty/status.rb +2 -1
- data/lib/faulty/storage.rb +3 -0
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +50 -55
- data/lib/faulty/storage/interface.rb +2 -1
- data/lib/faulty/storage/memory.rb +7 -3
- data/lib/faulty/storage/redis.rb +69 -7
- data/lib/faulty/version.rb +1 -1
- metadata +15 -20
- data/.travis.yml +0 -46
@@ -14,7 +14,8 @@ class Faulty
|
|
14
14
|
# @param circuit [Circuit] The circuit that ran
|
15
15
|
# @param time [Integer] The unix timestamp for the run
|
16
16
|
# @param success [Boolean] True if the run succeeded
|
17
|
-
# @return [
|
17
|
+
# @return [Array<Array>] An array of the new history tuples after adding
|
18
|
+
# the new entry, see {#history}
|
18
19
|
def entry(circuit, time, success)
|
19
20
|
raise NotImplementedError
|
20
21
|
end
|
@@ -4,8 +4,9 @@ class Faulty
|
|
4
4
|
module Storage
|
5
5
|
# The default in-memory storage for circuits
|
6
6
|
#
|
7
|
-
# This implementation is
|
8
|
-
#
|
7
|
+
# This implementation is thread-safe and circuit state is shared across
|
8
|
+
# threads. Since state is stored in-memory, this state is not shared across
|
9
|
+
# processes, or persisted across application restarts.
|
9
10
|
#
|
10
11
|
# Circuit state and runs are stored in memory. Although runs have a maximum
|
11
12
|
# size within a circuit, there is no limit on the number of circuits that
|
@@ -18,6 +19,9 @@ class Faulty
|
|
18
19
|
#
|
19
20
|
# This can be used as a reference implementation for storage backends that
|
20
21
|
# store a list of circuit run entries.
|
22
|
+
#
|
23
|
+
# @todo Add a more sophsticated implmentation that can limit the number of
|
24
|
+
# circuits stored.
|
21
25
|
class Memory
|
22
26
|
attr_reader :options
|
23
27
|
|
@@ -85,7 +89,7 @@ class Faulty
|
|
85
89
|
runs.push([time, success])
|
86
90
|
runs.shift if runs.size > options.max_sample_size
|
87
91
|
end
|
88
|
-
memory.
|
92
|
+
memory.runs.value
|
89
93
|
end
|
90
94
|
|
91
95
|
# Mark a circuit as open
|
data/lib/faulty/storage/redis.rb
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
class Faulty
|
4
4
|
module Storage
|
5
|
+
# A storage backend for storing circuit state in Redis.
|
6
|
+
#
|
7
|
+
# When using this or any networked backend, be sure to evaluate the risk,
|
8
|
+
# and set conservative timeouts so that the circuit storage does not cause
|
9
|
+
# cascading failures in your application when evaluating circuits. Always
|
10
|
+
# wrap this backend with a {FaultTolerantProxy} to limit the effect of
|
11
|
+
# these types of events.
|
5
12
|
class Redis # rubocop:disable Metrics/ClassLength
|
6
13
|
# Separates the time/status for history entry strings
|
7
14
|
ENTRY_SEPARATOR = ':'
|
@@ -27,12 +34,17 @@ class Faulty
|
|
27
34
|
# circuit run history entry. Default `100`.
|
28
35
|
# @!attribute [r] circuit_ttl
|
29
36
|
# @return [Integer] The maximum number of seconds to keep a circuit.
|
30
|
-
# A value of `nil` disables circuit expiration.
|
37
|
+
# A value of `nil` disables circuit expiration. This does not apply to
|
38
|
+
# locks, which have an indefinite storage time.
|
31
39
|
# Default `604_800` (1 week).
|
32
40
|
# @!attribute [r] list_granularity
|
33
41
|
# @return [Integer] The number of seconds after which a new set is
|
34
42
|
# created to store circuit names. The old set is kept until
|
35
43
|
# circuit_ttl expires. Default `3600` (1 hour).
|
44
|
+
# @!attribute [r] disable_warnings
|
45
|
+
# @return [Boolean] By default, this class warns if the client options
|
46
|
+
# are outside the recommended values. Set to true to disable these
|
47
|
+
# warnings.
|
36
48
|
Options = Struct.new(
|
37
49
|
:client,
|
38
50
|
:key_prefix,
|
@@ -40,7 +52,8 @@ class Faulty
|
|
40
52
|
:max_sample_size,
|
41
53
|
:sample_ttl,
|
42
54
|
:circuit_ttl,
|
43
|
-
:list_granularity
|
55
|
+
:list_granularity,
|
56
|
+
:disable_warnings
|
44
57
|
) do
|
45
58
|
include ImmutableOptions
|
46
59
|
|
@@ -53,7 +66,8 @@ class Faulty
|
|
53
66
|
max_sample_size: 100,
|
54
67
|
sample_ttl: 1800,
|
55
68
|
circuit_ttl: 604_800,
|
56
|
-
list_granularity: 3600
|
69
|
+
list_granularity: 3600,
|
70
|
+
disable_warnings: false
|
57
71
|
}
|
58
72
|
end
|
59
73
|
|
@@ -62,7 +76,7 @@ class Faulty
|
|
62
76
|
end
|
63
77
|
|
64
78
|
def finalize
|
65
|
-
self.client = ::Redis.new unless client
|
79
|
+
self.client = ::Redis.new(timeout: 1) unless client
|
66
80
|
end
|
67
81
|
end
|
68
82
|
|
@@ -70,6 +84,8 @@ class Faulty
|
|
70
84
|
# @yield [Options] For setting options in a block
|
71
85
|
def initialize(**options, &block)
|
72
86
|
@options = Options.new(options, &block)
|
87
|
+
|
88
|
+
check_client_options!
|
73
89
|
end
|
74
90
|
|
75
91
|
# Add an entry to storage
|
@@ -79,15 +95,15 @@ class Faulty
|
|
79
95
|
# @return (see Interface#entry)
|
80
96
|
def entry(circuit, time, success)
|
81
97
|
key = entries_key(circuit)
|
82
|
-
pipe do |r|
|
98
|
+
result = pipe do |r|
|
83
99
|
r.sadd(list_key, circuit.name)
|
84
100
|
r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
|
85
101
|
r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
|
86
102
|
r.ltrim(key, 0, options.max_sample_size - 1)
|
87
103
|
r.expire(key, options.sample_ttl) if options.sample_ttl
|
104
|
+
r.lrange(key, 0, -1)
|
88
105
|
end
|
89
|
-
|
90
|
-
status(circuit)
|
106
|
+
map_entries(result.last)
|
91
107
|
end
|
92
108
|
|
93
109
|
# Mark a circuit as open
|
@@ -196,6 +212,9 @@ class Faulty
|
|
196
212
|
map_entries(entries).reverse
|
197
213
|
end
|
198
214
|
|
215
|
+
# List all unexpired circuits
|
216
|
+
#
|
217
|
+
# @return (see Interface#list)
|
199
218
|
def list
|
200
219
|
redis { |r| r.sunion(*all_list_keys) }
|
201
220
|
end
|
@@ -330,6 +349,49 @@ class Faulty
|
|
330
349
|
[time.to_i, state == '1']
|
331
350
|
end
|
332
351
|
end
|
352
|
+
|
353
|
+
def check_client_options!
|
354
|
+
return if options.disable_warnings
|
355
|
+
|
356
|
+
check_redis_options!
|
357
|
+
check_pool_options!
|
358
|
+
rescue StandardError => e
|
359
|
+
warn "Faulty error while checking client options: #{e.message}"
|
360
|
+
end
|
361
|
+
|
362
|
+
def check_redis_options!
|
363
|
+
ropts = redis { |r| r.instance_variable_get(:@client).options }
|
364
|
+
|
365
|
+
bad_timeouts = {}
|
366
|
+
%i[connect_timeout read_timeout write_timeout].each do |time_opt|
|
367
|
+
bad_timeouts[time_opt] = ropts[time_opt] if ropts[time_opt] > 2
|
368
|
+
end
|
369
|
+
|
370
|
+
unless bad_timeouts.empty?
|
371
|
+
warn <<~MSG
|
372
|
+
Faulty recommends setting Redis timeouts <= 2 to prevent cascading
|
373
|
+
failures when evaluating circuits. Your options are:
|
374
|
+
#{bad_timeouts}
|
375
|
+
MSG
|
376
|
+
end
|
377
|
+
|
378
|
+
if ropts[:reconnect_attempts] > 1
|
379
|
+
warn <<~MSG
|
380
|
+
Faulty recommends setting Redis reconnect_attempts to <= 1 to
|
381
|
+
prevent cascading failures. Your setting is #{ropts[:reconnect_attempts]}
|
382
|
+
MSG
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
def check_pool_options!
|
387
|
+
if options.client.class.name == 'ConnectionPool'
|
388
|
+
timeout = options.client.instance_variable_get(:@timeout)
|
389
|
+
warn(<<~MSG) if timeout > 2
|
390
|
+
Faulty recommends setting ConnectionPool timeouts <= 2 to prevent
|
391
|
+
cascading failures when evaluating circuits. Your setting is #{timeout}
|
392
|
+
MSG
|
393
|
+
end
|
394
|
+
end
|
333
395
|
end
|
334
396
|
end
|
335
397
|
end
|
data/lib/faulty/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: faulty
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Howard
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -38,32 +38,18 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '2.0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: honeybadger
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '2.0'
|
48
|
-
type: :development
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '2.0'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: redis
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
|
-
- - "
|
45
|
+
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
47
|
version: '3.0'
|
62
48
|
type: :development
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
|
-
- - "
|
52
|
+
- - ">="
|
67
53
|
- !ruby/object:Gem::Version
|
68
54
|
version: '3.0'
|
69
55
|
- !ruby/object:Gem::Dependency
|
@@ -129,10 +115,10 @@ executables: []
|
|
129
115
|
extensions: []
|
130
116
|
extra_rdoc_files: []
|
131
117
|
files:
|
118
|
+
- ".github/workflows/ci.yml"
|
132
119
|
- ".gitignore"
|
133
120
|
- ".rspec"
|
134
121
|
- ".rubocop.yml"
|
135
|
-
- ".travis.yml"
|
136
122
|
- ".yardopts"
|
137
123
|
- CHANGELOG.md
|
138
124
|
- Gemfile
|
@@ -148,6 +134,8 @@ files:
|
|
148
134
|
- faulty.gemspec
|
149
135
|
- lib/faulty.rb
|
150
136
|
- lib/faulty/cache.rb
|
137
|
+
- lib/faulty/cache/auto_wire.rb
|
138
|
+
- lib/faulty/cache/circuit_proxy.rb
|
151
139
|
- lib/faulty/cache/default.rb
|
152
140
|
- lib/faulty/cache/fault_tolerant_proxy.rb
|
153
141
|
- lib/faulty/cache/interface.rb
|
@@ -163,9 +151,16 @@ files:
|
|
163
151
|
- lib/faulty/events/log_listener.rb
|
164
152
|
- lib/faulty/events/notifier.rb
|
165
153
|
- lib/faulty/immutable_options.rb
|
154
|
+
- lib/faulty/patch.rb
|
155
|
+
- lib/faulty/patch/base.rb
|
156
|
+
- lib/faulty/patch/mysql2.rb
|
157
|
+
- lib/faulty/patch/redis.rb
|
166
158
|
- lib/faulty/result.rb
|
167
159
|
- lib/faulty/status.rb
|
168
160
|
- lib/faulty/storage.rb
|
161
|
+
- lib/faulty/storage/auto_wire.rb
|
162
|
+
- lib/faulty/storage/circuit_proxy.rb
|
163
|
+
- lib/faulty/storage/fallback_chain.rb
|
169
164
|
- lib/faulty/storage/fault_tolerant_proxy.rb
|
170
165
|
- lib/faulty/storage/interface.rb
|
171
166
|
- lib/faulty/storage/memory.rb
|
@@ -190,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
190
185
|
- !ruby/object:Gem::Version
|
191
186
|
version: '0'
|
192
187
|
requirements: []
|
193
|
-
rubygems_version: 3.
|
188
|
+
rubygems_version: 3.1.2
|
194
189
|
signing_key:
|
195
190
|
specification_version: 4
|
196
191
|
summary: Fault-tolerance tools for ruby based on circuit-breakers
|
data/.travis.yml
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
---
|
2
|
-
language: ruby
|
3
|
-
rvm:
|
4
|
-
- 2.3
|
5
|
-
- 2.4
|
6
|
-
- 2.5
|
7
|
-
- 2.6
|
8
|
-
- 2.7
|
9
|
-
- jruby
|
10
|
-
- truffleruby
|
11
|
-
|
12
|
-
env:
|
13
|
-
global:
|
14
|
-
- COVERAGE=1
|
15
|
-
# Encrypted code climate CC_TEST_REPORTER_ID=
|
16
|
-
- secure: 'a/qmESTnDq37KvBY4u1VcBEjd7Y3b16L28AiV0y3d7SbrGeuWPmTR7WMvD8b0CV1Ou5wpic9HAxC9u2ZYhy0/BN9I4AOnkyj4IXF9K+ETbTq2XOAix38hNF4a/Hl89mgKD3IPtjxVrQ83h7pxbqtQ2jPnLyiR7mTj0rfOmzTSaItcn/auONuvanHuitGjYTw3xUA1okbhMy8cVh3nbozP0m3eujSbSOpLD3N9+s+A8lE+6VbNvoygqoqdcQqlYH4q5x+SGaD7L1illIywxVWpExygB9jIum3mhaXe7ThJ/FkUfCHqINRejR63QDVwvYGRPnrlg0c8rNiLDjfVyY83ESSqXN1hf/vzyVtGruj4t/35VsFL/fyE/iNGDDmMu/GH5Gbsa9+lykeRSOL97VAlzAuVeb4AwvZ9HFV5QcbGnNkrVG8sTMIOcUzJVGX2C38fnElGxGwrbpV8+ReMZtcQZSg2isRRKDUFRBdFFUrHKK8uqII+a5oZDi5OK7ytZHg8XTbrpJXStYhQjRMfbHsd2YFGeiyLWqzrASd/bpgdyLtZuQ1/r7z1IVGJjb01nNhD4mchU3Lj4mxaQe2zvBEhbUZufz2zrIDPiKxsQvYcqw4nvHJE7zWM0cMgHvQrzcZH44sHVtdVavjSPoOzN6665zaA9sCcYffESCPPQTnopE='
|
17
|
-
|
18
|
-
services:
|
19
|
-
- redis-server
|
20
|
-
|
21
|
-
before_script:
|
22
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
23
|
-
- chmod +x ./cc-test-reporter
|
24
|
-
- ./cc-test-reporter before-build
|
25
|
-
|
26
|
-
script:
|
27
|
-
- bundle exec rubocop
|
28
|
-
- bundle exec rspec --format doc
|
29
|
-
- bin/check-version
|
30
|
-
|
31
|
-
after_script:
|
32
|
-
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|
33
|
-
|
34
|
-
jobs:
|
35
|
-
include:
|
36
|
-
- stage: release
|
37
|
-
rvm: 2.7
|
38
|
-
script: skip
|
39
|
-
deploy:
|
40
|
-
provider: rubygems
|
41
|
-
api_key:
|
42
|
-
secure: 'eS6u+XyT6hHk/0gLXWzoe3RTJzEVFQHcJ+MNGSp7iq+cavJHisndcYBmUxi6/Ttb/aT/YoczhIWYo6WPbcjDqWszDfUsrQyaiOiZy4BBcMwN1WHeHkeRHfO2NX7us0AKO66TAiNfpMqUR2UT/c1LCPtLb+bkG6IFWxRuF5Fo32e0th6Z+hJ4Se2E/Qg1lrZk5zBlhQSOtU88vWQAkT9FdzpwBskVENBuDmH5YTV2Y28QYHsDtSNIxoDiUK9LOoPxaYUQp5ZZ58HZHtPdNydPHvtOXaWcBlakH5HNkh3FUHsxqbA1b3U412PC4TK+0jnxfISH4EOAMZEkL1nmroJORlb5nlwG1eiHpPVbOke1z2cZarGmWWAEf9bGE99GVbuxUxRrm9i1rmPdJY2G6Z0Kz8zoLTDYI/9l2P81/99a7h84rWACeeI3bcdvqViLxUuVMiwQjQsOZhzfq1M6jjETAWAI3AMRLxaGSgp0LV3WtaSWk1T5qvOvOF2HIfQ1EMd74kblJrVGWUiqd94/UgQoB/+lg9PdP/h4aHY5Cq1ec83wzaO6leKNlO+EQfyAVD1nd8kxo7YHEUpcB8wWCNFA5iaLqwIMt7aeWvG/BVUIH0apppzJGDy9UZ0gGsYi6ID3gbRRgmHIdJospr6TN7hksu1ZqkwEaprpXq8zH0KFYW0='
|
43
|
-
gem: faulty
|
44
|
-
on:
|
45
|
-
tags: true
|
46
|
-
repo: ParentSquare/faulty
|