faulty 0.2.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +56 -0
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +50 -0
- data/Gemfile +11 -3
- data/README.md +919 -303
- data/bin/check-version +5 -1
- data/faulty.gemspec +1 -2
- data/lib/faulty.rb +35 -23
- data/lib/faulty/cache.rb +2 -0
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +9 -20
- data/lib/faulty/cache/fault_tolerant_proxy.rb +13 -2
- data/lib/faulty/cache/rails.rb +8 -9
- data/lib/faulty/circuit.rb +30 -15
- data/lib/faulty/error.rb +25 -3
- data/lib/faulty/events/log_listener.rb +4 -5
- data/lib/faulty/patch.rb +154 -0
- data/lib/faulty/patch/base.rb +46 -0
- data/lib/faulty/patch/mysql2.rb +81 -0
- data/lib/faulty/patch/redis.rb +93 -0
- data/lib/faulty/status.rb +2 -1
- data/lib/faulty/storage.rb +3 -0
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +50 -55
- data/lib/faulty/storage/interface.rb +2 -1
- data/lib/faulty/storage/memory.rb +7 -3
- data/lib/faulty/storage/redis.rb +69 -7
- data/lib/faulty/version.rb +1 -1
- metadata +15 -20
- data/.travis.yml +0 -46
@@ -14,7 +14,8 @@ class Faulty
|
|
14
14
|
# @param circuit [Circuit] The circuit that ran
|
15
15
|
# @param time [Integer] The unix timestamp for the run
|
16
16
|
# @param success [Boolean] True if the run succeeded
|
17
|
-
# @return [
|
17
|
+
# @return [Array<Array>] An array of the new history tuples after adding
|
18
|
+
# the new entry, see {#history}
|
18
19
|
def entry(circuit, time, success)
|
19
20
|
raise NotImplementedError
|
20
21
|
end
|
@@ -4,8 +4,9 @@ class Faulty
|
|
4
4
|
module Storage
|
5
5
|
# The default in-memory storage for circuits
|
6
6
|
#
|
7
|
-
# This implementation is
|
8
|
-
#
|
7
|
+
# This implementation is thread-safe and circuit state is shared across
|
8
|
+
# threads. Since state is stored in-memory, this state is not shared across
|
9
|
+
# processes, or persisted across application restarts.
|
9
10
|
#
|
10
11
|
# Circuit state and runs are stored in memory. Although runs have a maximum
|
11
12
|
# size within a circuit, there is no limit on the number of circuits that
|
@@ -18,6 +19,9 @@ class Faulty
|
|
18
19
|
#
|
19
20
|
# This can be used as a reference implementation for storage backends that
|
20
21
|
# store a list of circuit run entries.
|
22
|
+
#
|
23
|
+
# @todo Add a more sophsticated implmentation that can limit the number of
|
24
|
+
# circuits stored.
|
21
25
|
class Memory
|
22
26
|
attr_reader :options
|
23
27
|
|
@@ -85,7 +89,7 @@ class Faulty
|
|
85
89
|
runs.push([time, success])
|
86
90
|
runs.shift if runs.size > options.max_sample_size
|
87
91
|
end
|
88
|
-
memory.
|
92
|
+
memory.runs.value
|
89
93
|
end
|
90
94
|
|
91
95
|
# Mark a circuit as open
|
data/lib/faulty/storage/redis.rb
CHANGED
@@ -2,6 +2,13 @@
|
|
2
2
|
|
3
3
|
class Faulty
|
4
4
|
module Storage
|
5
|
+
# A storage backend for storing circuit state in Redis.
|
6
|
+
#
|
7
|
+
# When using this or any networked backend, be sure to evaluate the risk,
|
8
|
+
# and set conservative timeouts so that the circuit storage does not cause
|
9
|
+
# cascading failures in your application when evaluating circuits. Always
|
10
|
+
# wrap this backend with a {FaultTolerantProxy} to limit the effect of
|
11
|
+
# these types of events.
|
5
12
|
class Redis # rubocop:disable Metrics/ClassLength
|
6
13
|
# Separates the time/status for history entry strings
|
7
14
|
ENTRY_SEPARATOR = ':'
|
@@ -27,12 +34,17 @@ class Faulty
|
|
27
34
|
# circuit run history entry. Default `100`.
|
28
35
|
# @!attribute [r] circuit_ttl
|
29
36
|
# @return [Integer] The maximum number of seconds to keep a circuit.
|
30
|
-
# A value of `nil` disables circuit expiration.
|
37
|
+
# A value of `nil` disables circuit expiration. This does not apply to
|
38
|
+
# locks, which have an indefinite storage time.
|
31
39
|
# Default `604_800` (1 week).
|
32
40
|
# @!attribute [r] list_granularity
|
33
41
|
# @return [Integer] The number of seconds after which a new set is
|
34
42
|
# created to store circuit names. The old set is kept until
|
35
43
|
# circuit_ttl expires. Default `3600` (1 hour).
|
44
|
+
# @!attribute [r] disable_warnings
|
45
|
+
# @return [Boolean] By default, this class warns if the client options
|
46
|
+
# are outside the recommended values. Set to true to disable these
|
47
|
+
# warnings.
|
36
48
|
Options = Struct.new(
|
37
49
|
:client,
|
38
50
|
:key_prefix,
|
@@ -40,7 +52,8 @@ class Faulty
|
|
40
52
|
:max_sample_size,
|
41
53
|
:sample_ttl,
|
42
54
|
:circuit_ttl,
|
43
|
-
:list_granularity
|
55
|
+
:list_granularity,
|
56
|
+
:disable_warnings
|
44
57
|
) do
|
45
58
|
include ImmutableOptions
|
46
59
|
|
@@ -53,7 +66,8 @@ class Faulty
|
|
53
66
|
max_sample_size: 100,
|
54
67
|
sample_ttl: 1800,
|
55
68
|
circuit_ttl: 604_800,
|
56
|
-
list_granularity: 3600
|
69
|
+
list_granularity: 3600,
|
70
|
+
disable_warnings: false
|
57
71
|
}
|
58
72
|
end
|
59
73
|
|
@@ -62,7 +76,7 @@ class Faulty
|
|
62
76
|
end
|
63
77
|
|
64
78
|
def finalize
|
65
|
-
self.client = ::Redis.new unless client
|
79
|
+
self.client = ::Redis.new(timeout: 1) unless client
|
66
80
|
end
|
67
81
|
end
|
68
82
|
|
@@ -70,6 +84,8 @@ class Faulty
|
|
70
84
|
# @yield [Options] For setting options in a block
|
71
85
|
def initialize(**options, &block)
|
72
86
|
@options = Options.new(options, &block)
|
87
|
+
|
88
|
+
check_client_options!
|
73
89
|
end
|
74
90
|
|
75
91
|
# Add an entry to storage
|
@@ -79,15 +95,15 @@ class Faulty
|
|
79
95
|
# @return (see Interface#entry)
|
80
96
|
def entry(circuit, time, success)
|
81
97
|
key = entries_key(circuit)
|
82
|
-
pipe do |r|
|
98
|
+
result = pipe do |r|
|
83
99
|
r.sadd(list_key, circuit.name)
|
84
100
|
r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
|
85
101
|
r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
|
86
102
|
r.ltrim(key, 0, options.max_sample_size - 1)
|
87
103
|
r.expire(key, options.sample_ttl) if options.sample_ttl
|
104
|
+
r.lrange(key, 0, -1)
|
88
105
|
end
|
89
|
-
|
90
|
-
status(circuit)
|
106
|
+
map_entries(result.last)
|
91
107
|
end
|
92
108
|
|
93
109
|
# Mark a circuit as open
|
@@ -196,6 +212,9 @@ class Faulty
|
|
196
212
|
map_entries(entries).reverse
|
197
213
|
end
|
198
214
|
|
215
|
+
# List all unexpired circuits
|
216
|
+
#
|
217
|
+
# @return (see Interface#list)
|
199
218
|
def list
|
200
219
|
redis { |r| r.sunion(*all_list_keys) }
|
201
220
|
end
|
@@ -330,6 +349,49 @@ class Faulty
|
|
330
349
|
[time.to_i, state == '1']
|
331
350
|
end
|
332
351
|
end
|
352
|
+
|
353
|
+
def check_client_options!
|
354
|
+
return if options.disable_warnings
|
355
|
+
|
356
|
+
check_redis_options!
|
357
|
+
check_pool_options!
|
358
|
+
rescue StandardError => e
|
359
|
+
warn "Faulty error while checking client options: #{e.message}"
|
360
|
+
end
|
361
|
+
|
362
|
+
def check_redis_options!
|
363
|
+
ropts = redis { |r| r.instance_variable_get(:@client).options }
|
364
|
+
|
365
|
+
bad_timeouts = {}
|
366
|
+
%i[connect_timeout read_timeout write_timeout].each do |time_opt|
|
367
|
+
bad_timeouts[time_opt] = ropts[time_opt] if ropts[time_opt] > 2
|
368
|
+
end
|
369
|
+
|
370
|
+
unless bad_timeouts.empty?
|
371
|
+
warn <<~MSG
|
372
|
+
Faulty recommends setting Redis timeouts <= 2 to prevent cascading
|
373
|
+
failures when evaluating circuits. Your options are:
|
374
|
+
#{bad_timeouts}
|
375
|
+
MSG
|
376
|
+
end
|
377
|
+
|
378
|
+
if ropts[:reconnect_attempts] > 1
|
379
|
+
warn <<~MSG
|
380
|
+
Faulty recommends setting Redis reconnect_attempts to <= 1 to
|
381
|
+
prevent cascading failures. Your setting is #{ropts[:reconnect_attempts]}
|
382
|
+
MSG
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
def check_pool_options!
|
387
|
+
if options.client.class.name == 'ConnectionPool'
|
388
|
+
timeout = options.client.instance_variable_get(:@timeout)
|
389
|
+
warn(<<~MSG) if timeout > 2
|
390
|
+
Faulty recommends setting ConnectionPool timeouts <= 2 to prevent
|
391
|
+
cascading failures when evaluating circuits. Your setting is #{timeout}
|
392
|
+
MSG
|
393
|
+
end
|
394
|
+
end
|
333
395
|
end
|
334
396
|
end
|
335
397
|
end
|
data/lib/faulty/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: faulty
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Howard
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-06-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -38,32 +38,18 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '2.0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: honeybadger
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '2.0'
|
48
|
-
type: :development
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '2.0'
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: redis
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
|
-
- - "
|
45
|
+
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
47
|
version: '3.0'
|
62
48
|
type: :development
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
|
-
- - "
|
52
|
+
- - ">="
|
67
53
|
- !ruby/object:Gem::Version
|
68
54
|
version: '3.0'
|
69
55
|
- !ruby/object:Gem::Dependency
|
@@ -129,10 +115,10 @@ executables: []
|
|
129
115
|
extensions: []
|
130
116
|
extra_rdoc_files: []
|
131
117
|
files:
|
118
|
+
- ".github/workflows/ci.yml"
|
132
119
|
- ".gitignore"
|
133
120
|
- ".rspec"
|
134
121
|
- ".rubocop.yml"
|
135
|
-
- ".travis.yml"
|
136
122
|
- ".yardopts"
|
137
123
|
- CHANGELOG.md
|
138
124
|
- Gemfile
|
@@ -148,6 +134,8 @@ files:
|
|
148
134
|
- faulty.gemspec
|
149
135
|
- lib/faulty.rb
|
150
136
|
- lib/faulty/cache.rb
|
137
|
+
- lib/faulty/cache/auto_wire.rb
|
138
|
+
- lib/faulty/cache/circuit_proxy.rb
|
151
139
|
- lib/faulty/cache/default.rb
|
152
140
|
- lib/faulty/cache/fault_tolerant_proxy.rb
|
153
141
|
- lib/faulty/cache/interface.rb
|
@@ -163,9 +151,16 @@ files:
|
|
163
151
|
- lib/faulty/events/log_listener.rb
|
164
152
|
- lib/faulty/events/notifier.rb
|
165
153
|
- lib/faulty/immutable_options.rb
|
154
|
+
- lib/faulty/patch.rb
|
155
|
+
- lib/faulty/patch/base.rb
|
156
|
+
- lib/faulty/patch/mysql2.rb
|
157
|
+
- lib/faulty/patch/redis.rb
|
166
158
|
- lib/faulty/result.rb
|
167
159
|
- lib/faulty/status.rb
|
168
160
|
- lib/faulty/storage.rb
|
161
|
+
- lib/faulty/storage/auto_wire.rb
|
162
|
+
- lib/faulty/storage/circuit_proxy.rb
|
163
|
+
- lib/faulty/storage/fallback_chain.rb
|
169
164
|
- lib/faulty/storage/fault_tolerant_proxy.rb
|
170
165
|
- lib/faulty/storage/interface.rb
|
171
166
|
- lib/faulty/storage/memory.rb
|
@@ -190,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
190
185
|
- !ruby/object:Gem::Version
|
191
186
|
version: '0'
|
192
187
|
requirements: []
|
193
|
-
rubygems_version: 3.
|
188
|
+
rubygems_version: 3.1.2
|
194
189
|
signing_key:
|
195
190
|
specification_version: 4
|
196
191
|
summary: Fault-tolerance tools for ruby based on circuit-breakers
|
data/.travis.yml
DELETED
@@ -1,46 +0,0 @@
|
|
1
|
-
---
|
2
|
-
language: ruby
|
3
|
-
rvm:
|
4
|
-
- 2.3
|
5
|
-
- 2.4
|
6
|
-
- 2.5
|
7
|
-
- 2.6
|
8
|
-
- 2.7
|
9
|
-
- jruby
|
10
|
-
- truffleruby
|
11
|
-
|
12
|
-
env:
|
13
|
-
global:
|
14
|
-
- COVERAGE=1
|
15
|
-
# Encrypted code climate CC_TEST_REPORTER_ID=
|
16
|
-
- secure: 'a/qmESTnDq37KvBY4u1VcBEjd7Y3b16L28AiV0y3d7SbrGeuWPmTR7WMvD8b0CV1Ou5wpic9HAxC9u2ZYhy0/BN9I4AOnkyj4IXF9K+ETbTq2XOAix38hNF4a/Hl89mgKD3IPtjxVrQ83h7pxbqtQ2jPnLyiR7mTj0rfOmzTSaItcn/auONuvanHuitGjYTw3xUA1okbhMy8cVh3nbozP0m3eujSbSOpLD3N9+s+A8lE+6VbNvoygqoqdcQqlYH4q5x+SGaD7L1illIywxVWpExygB9jIum3mhaXe7ThJ/FkUfCHqINRejR63QDVwvYGRPnrlg0c8rNiLDjfVyY83ESSqXN1hf/vzyVtGruj4t/35VsFL/fyE/iNGDDmMu/GH5Gbsa9+lykeRSOL97VAlzAuVeb4AwvZ9HFV5QcbGnNkrVG8sTMIOcUzJVGX2C38fnElGxGwrbpV8+ReMZtcQZSg2isRRKDUFRBdFFUrHKK8uqII+a5oZDi5OK7ytZHg8XTbrpJXStYhQjRMfbHsd2YFGeiyLWqzrASd/bpgdyLtZuQ1/r7z1IVGJjb01nNhD4mchU3Lj4mxaQe2zvBEhbUZufz2zrIDPiKxsQvYcqw4nvHJE7zWM0cMgHvQrzcZH44sHVtdVavjSPoOzN6665zaA9sCcYffESCPPQTnopE='
|
17
|
-
|
18
|
-
services:
|
19
|
-
- redis-server
|
20
|
-
|
21
|
-
before_script:
|
22
|
-
- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
23
|
-
- chmod +x ./cc-test-reporter
|
24
|
-
- ./cc-test-reporter before-build
|
25
|
-
|
26
|
-
script:
|
27
|
-
- bundle exec rubocop
|
28
|
-
- bundle exec rspec --format doc
|
29
|
-
- bin/check-version
|
30
|
-
|
31
|
-
after_script:
|
32
|
-
- ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
|
33
|
-
|
34
|
-
jobs:
|
35
|
-
include:
|
36
|
-
- stage: release
|
37
|
-
rvm: 2.7
|
38
|
-
script: skip
|
39
|
-
deploy:
|
40
|
-
provider: rubygems
|
41
|
-
api_key:
|
42
|
-
secure: 'eS6u+XyT6hHk/0gLXWzoe3RTJzEVFQHcJ+MNGSp7iq+cavJHisndcYBmUxi6/Ttb/aT/YoczhIWYo6WPbcjDqWszDfUsrQyaiOiZy4BBcMwN1WHeHkeRHfO2NX7us0AKO66TAiNfpMqUR2UT/c1LCPtLb+bkG6IFWxRuF5Fo32e0th6Z+hJ4Se2E/Qg1lrZk5zBlhQSOtU88vWQAkT9FdzpwBskVENBuDmH5YTV2Y28QYHsDtSNIxoDiUK9LOoPxaYUQp5ZZ58HZHtPdNydPHvtOXaWcBlakH5HNkh3FUHsxqbA1b3U412PC4TK+0jnxfISH4EOAMZEkL1nmroJORlb5nlwG1eiHpPVbOke1z2cZarGmWWAEf9bGE99GVbuxUxRrm9i1rmPdJY2G6Z0Kz8zoLTDYI/9l2P81/99a7h84rWACeeI3bcdvqViLxUuVMiwQjQsOZhzfq1M6jjETAWAI3AMRLxaGSgp0LV3WtaSWk1T5qvOvOF2HIfQ1EMd74kblJrVGWUiqd94/UgQoB/+lg9PdP/h4aHY5Cq1ec83wzaO6leKNlO+EQfyAVD1nd8kxo7YHEUpcB8wWCNFA5iaLqwIMt7aeWvG/BVUIH0apppzJGDy9UZ0gGsYi6ID3gbRRgmHIdJospr6TN7hksu1ZqkwEaprpXq8zH0KFYW0='
|
43
|
-
gem: faulty
|
44
|
-
on:
|
45
|
-
tags: true
|
46
|
-
repo: ParentSquare/faulty
|