faulty 0.1.4 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +49 -0
  3. data/.rubocop.yml +9 -0
  4. data/CHANGELOG.md +55 -0
  5. data/Gemfile +8 -3
  6. data/README.md +883 -310
  7. data/bin/check-version +5 -1
  8. data/faulty.gemspec +1 -1
  9. data/lib/faulty.rb +167 -43
  10. data/lib/faulty/cache.rb +3 -1
  11. data/lib/faulty/cache/auto_wire.rb +58 -0
  12. data/lib/faulty/cache/circuit_proxy.rb +61 -0
  13. data/lib/faulty/cache/default.rb +10 -21
  14. data/lib/faulty/cache/fault_tolerant_proxy.rb +15 -4
  15. data/lib/faulty/cache/interface.rb +1 -1
  16. data/lib/faulty/cache/mock.rb +1 -1
  17. data/lib/faulty/cache/null.rb +1 -1
  18. data/lib/faulty/cache/rails.rb +9 -10
  19. data/lib/faulty/circuit.rb +31 -16
  20. data/lib/faulty/error.rb +29 -7
  21. data/lib/faulty/events.rb +1 -1
  22. data/lib/faulty/events/callback_listener.rb +1 -1
  23. data/lib/faulty/events/honeybadger_listener.rb +1 -1
  24. data/lib/faulty/events/listener_interface.rb +1 -1
  25. data/lib/faulty/events/log_listener.rb +5 -6
  26. data/lib/faulty/events/notifier.rb +1 -1
  27. data/lib/faulty/immutable_options.rb +1 -1
  28. data/lib/faulty/patch.rb +154 -0
  29. data/lib/faulty/patch/base.rb +46 -0
  30. data/lib/faulty/patch/redis.rb +60 -0
  31. data/lib/faulty/result.rb +2 -2
  32. data/lib/faulty/status.rb +3 -2
  33. data/lib/faulty/storage.rb +4 -1
  34. data/lib/faulty/storage/auto_wire.rb +107 -0
  35. data/lib/faulty/storage/circuit_proxy.rb +64 -0
  36. data/lib/faulty/storage/fallback_chain.rb +207 -0
  37. data/lib/faulty/storage/fault_tolerant_proxy.rb +52 -57
  38. data/lib/faulty/storage/interface.rb +3 -2
  39. data/lib/faulty/storage/memory.rb +8 -4
  40. data/lib/faulty/storage/redis.rb +75 -13
  41. data/lib/faulty/version.rb +2 -2
  42. metadata +14 -7
  43. data/.travis.yml +0 -46
  44. data/lib/faulty/scope.rb +0 -117
@@ -1,15 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Faulty
3
+ class Faulty
4
4
  module Storage
5
5
  # A wrapper for storage backends that may raise errors
6
6
  #
7
- # {Scope} automatically wraps all non-fault-tolerant storage backends with
7
+ # {Faulty#initialize} automatically wraps all non-fault-tolerant storage backends with
8
8
  # this class.
9
9
  #
10
10
  # If the storage backend raises a `StandardError`, it will be captured and
11
11
  # sent to the notifier.
12
12
  class FaultTolerantProxy
13
+ extend Forwardable
14
+
13
15
  attr_reader :options
14
16
 
15
17
  # Options for {FaultTolerantProxy}
@@ -36,6 +38,53 @@ module Faulty
36
38
  @options = Options.new(options, &block)
37
39
  end
38
40
 
41
+ # Wrap a storage backend in a FaultTolerantProxy unless it's already
42
+ # fault tolerant
43
+ #
44
+ # @param storage [Storage::Interface] The storage to maybe wrap
45
+ # @return [Storage::Interface] The original storage or a {FaultTolerantProxy}
46
+ def self.wrap(storage, **options, &block)
47
+ return storage if storage.fault_tolerant?
48
+
49
+ new(storage, **options, &block)
50
+ end
51
+
52
+ # @!method lock(circuit, state)
53
+ # Lock is not called in normal operation, so it doesn't capture errors
54
+ #
55
+ # @see Interface#lock
56
+ # @param (see Interface#lock)
57
+ # @return (see Interface#lock)
58
+ #
59
+ # @!method unlock(circuit)
60
+ # Unlock is not called in normal operation, so it doesn't capture errors
61
+ #
62
+ # @see Interface#unlock
63
+ # @param (see Interface#unlock)
64
+ # @return (see Interface#unlock)
65
+ #
66
+ # @!method reset(circuit)
67
+ # Reset is not called in normal operation, so it doesn't capture errors
68
+ #
69
+ # @see Interface#reset
70
+ # @param (see Interface#reset)
71
+ # @return (see Interface#reset)
72
+ #
73
+ # @!method history(circuit)
74
+ # History is not called in normal operation, so it doesn't capture errors
75
+ #
76
+ # @see Interface#history
77
+ # @param (see Interface#history)
78
+ # @return (see Interface#history)
79
+ #
80
+ # @!method list
81
+ # List is not called in normal operation, so it doesn't capture errors
82
+ #
83
+ # @see Interface#list
84
+ # @param (see Interface#list)
85
+ # @return (see Interface#list)
86
+ def_delegators :@storage, :lock, :unlock, :reset, :history, :list
87
+
39
88
  # Add a history entry safely
40
89
  #
41
90
  # @see Interface#entry
@@ -45,7 +94,7 @@ module Faulty
45
94
  @storage.entry(circuit, time, success)
46
95
  rescue StandardError => e
47
96
  options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e)
48
- stub_status(circuit)
97
+ []
49
98
  end
50
99
 
51
100
  # Safely mark a circuit as open
@@ -84,36 +133,6 @@ module Faulty
84
133
  false
85
134
  end
86
135
 
87
- # Since lock is not called in normal operation, it does not capture
88
- # errors
89
- #
90
- # @see Interface#lock
91
- # @param (see Interface#lock)
92
- # @return (see Interface#lock)
93
- def lock(circuit, state)
94
- @storage.lock(circuit, state)
95
- end
96
-
97
- # Since unlock is not called in normal operation, it does not capture
98
- # errors
99
- #
100
- # @see Interface#unlock
101
- # @param (see Interface#unlock)
102
- # @return (see Interface#unlock)
103
- def unlock(circuit)
104
- @storage.unlock(circuit)
105
- end
106
-
107
- # Since reset is not called in normal operation, it does not capture
108
- # errors
109
- #
110
- # @see Interface#reset
111
- # @param (see Interface#reset)
112
- # @return (see Interface#reset)
113
- def reset(circuit)
114
- @storage.reset(circuit)
115
- end
116
-
117
136
  # Safely get the status of a circuit
118
137
  #
119
138
  # If the backend is unavailable, this returns a stub status that
@@ -129,30 +148,6 @@ module Faulty
129
148
  stub_status(circuit)
130
149
  end
131
150
 
132
- # Since history is not called in normal operation, it does not capture
133
- # errors
134
- #
135
- # @see Interface#history
136
- # @param (see Interface#history)
137
- # @return (see Interface#history)
138
- def history(circuit)
139
- @storage.history(circuit)
140
- end
141
-
142
- # Safely get the list of circuit names
143
- #
144
- # If the backend is unavailable, this returns an empty array
145
- #
146
- # @see Interface#list
147
- # @param (see Interface#list)
148
- # @return (see Interface#list)
149
- def list
150
- @storage.list
151
- rescue StandardError => e
152
- options.notifier.notify(:storage_failure, action: :list, error: e)
153
- []
154
- end
155
-
156
151
  # This cache makes any storage fault tolerant, so this is always `true`
157
152
  #
158
153
  # @return [true]
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Faulty
3
+ class Faulty
4
4
  module Storage
5
5
  # The interface required for a storage backend implementation
6
6
  #
@@ -14,7 +14,8 @@ module Faulty
14
14
  # @param circuit [Circuit] The circuit that ran
15
15
  # @param time [Integer] The unix timestamp for the run
16
16
  # @param success [Boolean] True if the run succeeded
17
- # @return [Status] The circuit status after the run is added
17
+ # @return [Array<Array>] An array of the new history tuples after adding
18
+ # the new entry, see {#history}
18
19
  def entry(circuit, time, success)
19
20
  raise NotImplementedError
20
21
  end
@@ -1,11 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Faulty
3
+ class Faulty
4
4
  module Storage
5
5
  # The default in-memory storage for circuits
6
6
  #
7
- # This implementation is most suitable to single-process, low volume
8
- # usage. It is thread-safe and circuit state is shared across threads.
7
+ # This implementation is thread-safe and circuit state is shared across
8
+ # threads. Since state is stored in-memory, this state is not shared across
9
+ # processes, or persisted across application restarts.
9
10
  #
10
11
  # Circuit state and runs are stored in memory. Although runs have a maximum
11
12
  # size within a circuit, there is no limit on the number of circuits that
@@ -18,6 +19,9 @@ module Faulty
18
19
  #
19
20
  # This can be used as a reference implementation for storage backends that
20
21
  # store a list of circuit run entries.
22
+ #
23
+ # @todo Add a more sophsticated implmentation that can limit the number of
24
+ # circuits stored.
21
25
  class Memory
22
26
  attr_reader :options
23
27
 
@@ -85,7 +89,7 @@ module Faulty
85
89
  runs.push([time, success])
86
90
  runs.shift if runs.size > options.max_sample_size
87
91
  end
88
- memory.status(circuit.options)
92
+ memory.runs.value
89
93
  end
90
94
 
91
95
  # Mark a circuit as open
@@ -1,7 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Faulty
3
+ class Faulty
4
4
  module Storage
5
+ # A storage backend for storing circuit state in Redis.
6
+ #
7
+ # When using this or any networked backend, be sure to evaluate the risk,
8
+ # and set conservative timeouts so that the circuit storage does not cause
9
+ # cascading failures in your application when evaluating circuits. Always
10
+ # wrap this backend with a {FaultTolerantProxy} to limit the effect of
11
+ # these types of events.
5
12
  class Redis # rubocop:disable Metrics/ClassLength
6
13
  # Separates the time/status for history entry strings
7
14
  ENTRY_SEPARATOR = ':'
@@ -27,12 +34,17 @@ module Faulty
27
34
  # circuit run history entry. Default `100`.
28
35
  # @!attribute [r] circuit_ttl
29
36
  # @return [Integer] The maximum number of seconds to keep a circuit.
30
- # A value of `nil` disables circuit expiration.
37
+ # A value of `nil` disables circuit expiration. This does not apply to
38
+ # locks, which have an indefinite storage time.
31
39
  # Default `604_800` (1 week).
32
40
  # @!attribute [r] list_granularity
33
41
  # @return [Integer] The number of seconds after which a new set is
34
42
  # created to store circuit names. The old set is kept until
35
43
  # circuit_ttl expires. Default `3600` (1 hour).
44
+ # @!attribute [r] disable_warnings
45
+ # @return [Boolean] By default, this class warns if the client options
46
+ # are outside the recommended values. Set to true to disable these
47
+ # warnings.
36
48
  Options = Struct.new(
37
49
  :client,
38
50
  :key_prefix,
@@ -40,7 +52,8 @@ module Faulty
40
52
  :max_sample_size,
41
53
  :sample_ttl,
42
54
  :circuit_ttl,
43
- :list_granularity
55
+ :list_granularity,
56
+ :disable_warnings
44
57
  ) do
45
58
  include ImmutableOptions
46
59
 
@@ -53,7 +66,8 @@ module Faulty
53
66
  max_sample_size: 100,
54
67
  sample_ttl: 1800,
55
68
  circuit_ttl: 604_800,
56
- list_granularity: 3600
69
+ list_granularity: 3600,
70
+ disable_warnings: false
57
71
  }
58
72
  end
59
73
 
@@ -62,7 +76,7 @@ module Faulty
62
76
  end
63
77
 
64
78
  def finalize
65
- self.client = ::Redis.new unless client
79
+ self.client = ::Redis.new(timeout: 1) unless client
66
80
  end
67
81
  end
68
82
 
@@ -70,6 +84,8 @@ module Faulty
70
84
  # @yield [Options] For setting options in a block
71
85
  def initialize(**options, &block)
72
86
  @options = Options.new(options, &block)
87
+
88
+ check_client_options!
73
89
  end
74
90
 
75
91
  # Add an entry to storage
@@ -79,15 +95,15 @@ module Faulty
79
95
  # @return (see Interface#entry)
80
96
  def entry(circuit, time, success)
81
97
  key = entries_key(circuit)
82
- pipe do |r|
98
+ result = pipe do |r|
83
99
  r.sadd(list_key, circuit.name)
84
100
  r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
85
101
  r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
86
102
  r.ltrim(key, 0, options.max_sample_size - 1)
87
103
  r.expire(key, options.sample_ttl) if options.sample_ttl
104
+ r.lrange(key, 0, -1)
88
105
  end
89
-
90
- status(circuit)
106
+ map_entries(result.last)
91
107
  end
92
108
 
93
109
  # Mark a circuit as open
@@ -97,7 +113,7 @@ module Faulty
97
113
  # @return (see Interface#open)
98
114
  def open(circuit, opened_at)
99
115
  redis do |r|
100
- opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
116
+ opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open', ex: options.circuit_ttl)
101
117
  r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
102
118
  opened
103
119
  end
@@ -110,7 +126,7 @@ module Faulty
110
126
  # @return (see Interface#reopen)
111
127
  def reopen(circuit, opened_at, previous_opened_at)
112
128
  redis do |r|
113
- compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
129
+ compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at, ex: options.circuit_ttl)
114
130
  end
115
131
  end
116
132
 
@@ -121,7 +137,7 @@ module Faulty
121
137
  # @return (see Interface#close)
122
138
  def close(circuit)
123
139
  redis do |r|
124
- closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
140
+ closed = compare_and_set(r, state_key(circuit), ['open'], 'closed', ex: options.circuit_ttl)
125
141
  r.del(entries_key(circuit)) if closed
126
142
  closed
127
143
  end
@@ -196,6 +212,9 @@ module Faulty
196
212
  map_entries(entries).reverse
197
213
  end
198
214
 
215
+ # List all unexpired circuits
216
+ #
217
+ # @return (see Interface#list)
199
218
  def list
200
219
  redis { |r| r.sunion(*all_list_keys) }
201
220
  end
@@ -287,10 +306,10 @@ module Faulty
287
306
  # @param new [String] The new value to set if the compare passes
288
307
  # @return [Boolean] True if the value was set to `new`, false if the CAS
289
308
  # failed
290
- def compare_and_set(redis, key, old, new)
309
+ def compare_and_set(redis, key, old, new, ex:)
291
310
  redis.watch(key) do
292
311
  if old.include?(redis.get(key))
293
- result = redis.multi { |m| m.set(key, new) }
312
+ result = redis.multi { |m| m.set(key, new, ex: ex) }
294
313
  result && result[0] == 'OK'
295
314
  else
296
315
  redis.unwatch
@@ -330,6 +349,49 @@ module Faulty
330
349
  [time.to_i, state == '1']
331
350
  end
332
351
  end
352
+
353
+ def check_client_options!
354
+ return if options.disable_warnings
355
+
356
+ check_redis_options!
357
+ check_pool_options!
358
+ rescue StandardError => e
359
+ warn "Faulty error while checking client options: #{e.message}"
360
+ end
361
+
362
+ def check_redis_options!
363
+ ropts = redis { |r| r.instance_variable_get(:@client).options }
364
+
365
+ bad_timeouts = {}
366
+ %i[connect_timeout read_timeout write_timeout].each do |time_opt|
367
+ bad_timeouts[time_opt] = ropts[time_opt] if ropts[time_opt] > 2
368
+ end
369
+
370
+ unless bad_timeouts.empty?
371
+ warn <<~MSG
372
+ Faulty recommends setting Redis timeouts <= 2 to prevent cascading
373
+ failures when evaluating circuits. Your options are:
374
+ #{bad_timeouts}
375
+ MSG
376
+ end
377
+
378
+ if ropts[:reconnect_attempts] > 1
379
+ warn <<~MSG
380
+ Faulty recommends setting Redis reconnect_attempts to <= 1 to
381
+ prevent cascading failures. Your setting is #{ropts[:reconnect_attempts]}
382
+ MSG
383
+ end
384
+ end
385
+
386
+ def check_pool_options!
387
+ if options.client.class.name == 'ConnectionPool'
388
+ timeout = options.client.instance_variable_get(:@timeout)
389
+ warn(<<~MSG) if timeout > 2
390
+ Faulty recommends setting ConnectionPool timeouts <= 2 to prevent
391
+ cascading failures when evaluating circuits. Your setting is #{timeout}
392
+ MSG
393
+ end
394
+ end
333
395
  end
334
396
  end
335
397
  end
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module Faulty
3
+ class Faulty
4
4
  # The current Faulty version
5
5
  def self.version
6
- Gem::Version.new('0.1.4')
6
+ Gem::Version.new('0.5.1')
7
7
  end
8
8
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: faulty
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Howard
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-18 00:00:00.000000000 Z
11
+ date: 2021-05-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -56,14 +56,14 @@ dependencies:
56
56
  name: redis
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - "~>"
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '3.0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - "~>"
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '3.0'
69
69
  - !ruby/object:Gem::Dependency
@@ -129,10 +129,10 @@ executables: []
129
129
  extensions: []
130
130
  extra_rdoc_files: []
131
131
  files:
132
+ - ".github/workflows/ci.yml"
132
133
  - ".gitignore"
133
134
  - ".rspec"
134
135
  - ".rubocop.yml"
135
- - ".travis.yml"
136
136
  - ".yardopts"
137
137
  - CHANGELOG.md
138
138
  - Gemfile
@@ -148,6 +148,8 @@ files:
148
148
  - faulty.gemspec
149
149
  - lib/faulty.rb
150
150
  - lib/faulty/cache.rb
151
+ - lib/faulty/cache/auto_wire.rb
152
+ - lib/faulty/cache/circuit_proxy.rb
151
153
  - lib/faulty/cache/default.rb
152
154
  - lib/faulty/cache/fault_tolerant_proxy.rb
153
155
  - lib/faulty/cache/interface.rb
@@ -163,10 +165,15 @@ files:
163
165
  - lib/faulty/events/log_listener.rb
164
166
  - lib/faulty/events/notifier.rb
165
167
  - lib/faulty/immutable_options.rb
168
+ - lib/faulty/patch.rb
169
+ - lib/faulty/patch/base.rb
170
+ - lib/faulty/patch/redis.rb
166
171
  - lib/faulty/result.rb
167
- - lib/faulty/scope.rb
168
172
  - lib/faulty/status.rb
169
173
  - lib/faulty/storage.rb
174
+ - lib/faulty/storage/auto_wire.rb
175
+ - lib/faulty/storage/circuit_proxy.rb
176
+ - lib/faulty/storage/fallback_chain.rb
170
177
  - lib/faulty/storage/fault_tolerant_proxy.rb
171
178
  - lib/faulty/storage/interface.rb
172
179
  - lib/faulty/storage/memory.rb
@@ -191,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
191
198
  - !ruby/object:Gem::Version
192
199
  version: '0'
193
200
  requirements: []
194
- rubygems_version: 3.0.8
201
+ rubygems_version: 3.1.2
195
202
  signing_key:
196
203
  specification_version: 4
197
204
  summary: Fault-tolerance tools for ruby based on circuit-breakers