faulty 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Cache
5
+ # The interface required for a cache backend implementation
6
+ #
7
+ # This is for documentation only and is not loaded
8
+ class Interface
9
+ # Retrieve a value from the cache if available
10
+ #
11
+ # @param key [String] The cache key
12
+ # @raise If the cache backend encounters a failure
13
+ # @return [Object, nil] The object if present, otherwise nil
14
+ def read(key)
15
+ raise NotImplementedError
16
+ end
17
+
18
+ # Write a value to the cache
19
+ #
20
+ # This may be any object. It's up to the cache implementation to
21
+ # serialize if necessary or raise an error if unsupported.
22
+ #
23
+ # @param key [String] The cache key
24
+ # @param expires_in [Integer, nil] The number of seconds until this cache
25
+ # entry expires. If nil, no expiration is set.
26
+ # @param value [Object] The value to write to the cache
27
+ # @raise If the cache backend encounters a failure
28
+ # @return [void]
29
+ def write(key, value, expires_in: nil)
30
+ raise NotImplementedError
31
+ end
32
+
33
+ # Can this cache backend raise an error?
34
+ #
35
+ # If the cache backend returns false from this method, it will be wrapped
36
+ # in a {FaultTolerantProxy}, otherwise it will be used as-is.
37
+ #
38
+ # @return [Boolean] True if this cache backend is fault tolerant
39
+ def fault_tolerant?
40
+ raise NotImplementedError
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Cache
5
+ # A mock cache for testing
6
+ #
7
+ # This never clears expired values from memory, and should not be used
8
+ # in production applications. Instead, use a more robust implementation like
9
+ # `ActiveSupport::Cache::MemoryStore`.
10
+ class Mock
11
+ def initialize
12
+ @cache = {}
13
+ @expires = {}
14
+ end
15
+
16
+ # Read `key` from the cache
17
+ #
18
+ # @return [Object, nil] The value if present and not expired
19
+ def read(key)
20
+ return if @expires[key] && @expires[key] < Faulty.current_time
21
+
22
+ @cache[key]
23
+ end
24
+
25
+ # Write `key` to the cache with an optional expiration
26
+ #
27
+ # @return [void]
28
+ def write(key, value, expires_in: nil)
29
+ @cache[key] = value
30
+ @expires[key] = Faulty.current_time + expires_in unless expires_in.nil?
31
+ end
32
+
33
+ # @return [true]
34
+ def fault_tolerant?
35
+ true
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Cache
5
+ # A cache backend that does nothing
6
+ #
7
+ # All methods are stubs and do no caching
8
+ class Null
9
+ # @return [nil]
10
+ def read(_key)
11
+ end
12
+
13
+ # @return [void]
14
+ def write(_key, _value, expires_in: nil)
15
+ end
16
+
17
+ # @return [true]
18
+ def fault_tolerant?
19
+ true
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Cache
5
+ # A wrapper for a Rails or ActiveSupport cache
6
+ #
7
+ class Rails
8
+ # @param cache The Rails cache to wrap
9
+ # @param fault_tolerant [Boolean] Whether the Rails cache is
10
+ # fault_tolerant. See {#fault_tolerant?} for more details
11
+ def initialize(cache = ::Rails.cache, fault_tolerant: false)
12
+ @cache = cache
13
+ @fault_tolerant = fault_tolerant
14
+ end
15
+
16
+ # (see Interface#read)
17
+ def read(key)
18
+ @cache.read(key)
19
+ end
20
+
21
+ # (see Interface#read)
22
+ def write(key, value, expires_in: nil)
23
+ @cache.write(key, value, expires_in: expires_in)
24
+ end
25
+
26
+ # Although ActiveSupport cache implementations are fault-tolerant,
27
+ # Rails.cache is not guranteed to be fault tolerant. For this reason,
28
+ # we require the user of this class to explicitly mark this cache as
29
+ # fault-tolerant using the {#initialize} parameter.
30
+ #
31
+ # @return [Boolean]
32
+ def fault_tolerant?
33
+ @fault_tolerant
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,436 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ # Runs code protected by a circuit breaker
5
+ #
6
+ # https://www.martinfowler.com/bliki/CircuitBreaker.html
7
+ #
8
+ # A circuit is intended to protect against repeated calls to a failing
9
+ # external dependency. For example, a vendor API may be failing continuously.
10
+ # In that case, we trip the circuit breaker and stop calling that API for
11
+ # a specified cool-down period.
12
+ #
13
+ # Once the cool-down passes, we try the API again, and if it succeeds, we reset
14
+ # the circuit.
15
+ #
16
+ # Why isn't there a timeout option?
17
+ # -----------------
18
+ # Timeout is inherently unsafe, and
19
+ # should not be used blindly.
20
+ # See [Why Ruby's timeout is Dangerous](https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying).
21
+ #
22
+ # You should prefer a network timeout like `open_timeout` and `read_timeout`, or
23
+ # write your own code to periodically check how long it has been running.
24
+ # If you're sure you want ruby's generic Timeout, you can apply it yourself
25
+ # inside the circuit run block.
26
+ class Circuit # rubocop:disable Metrics/ClassLength
27
+ CACHE_REFRESH_SUFFIX = '.faulty_refresh'
28
+
29
+ attr_reader :name
30
+ attr_reader :options
31
+
32
+ # Options for {Circuit}
33
+ #
34
+ # @!attribute [r] cache_expires_in
35
+ # @return [Integer, nil] The number of seconds to keep
36
+ # cached results. A value of nil will keep the cache indefinitely.
37
+ # Default `86400`.
38
+ # @!attribute [r] cache_refreshes_after
39
+ # @return [Integer, nil] The number of seconds after which we attempt
40
+ # to refresh the cache even if it's not expired. If the circuit fails,
41
+ # we continue serving the value from cache until `cache_expires_in`.
42
+ # A value of `nil` disables cache refreshing.
43
+ # Default `900`.
44
+ # @!attribute [r] cache_refresh_jitter
45
+ # @return [Integer] The maximum number of seconds to randomly add or
46
+ # subtract from `cache_refreshes_after` when determining whether to
47
+ # refresh the cache. A non-zero value helps reduce a "thundering herd"
48
+ # cache refresh in most scenarios. Set to `0` to disable jitter.
49
+ # Default `0.2 * cache_refreshes_after`.
50
+ # @!attribute [r] cool_down
51
+ # @return [Integer] The number of seconds the circuit will
52
+ # stay open after it is tripped. Default 300.
53
+ # @!attribute [r] evaluation_window
54
+ # @return [Integer] The number of seconds of history that
55
+ # will be evaluated to determine the failure rate for a circuit.
56
+ # Default `60`.
57
+ # @!attribute [r] rate_threshold
58
+ # @return [Float] The minimum failure rate required to trip
59
+ # the circuit. For example, `0.5` requires at least a 50% failure rate to
60
+ # trip. Default `0.5`.
61
+ # @!attribute [r] sample_threshold
62
+ # @return [Integer] The minimum number of runs required before
63
+ # a circuit can trip. A value of 1 means that the circuit will trip
64
+ # immediately when a failure occurs. Default `3`.
65
+ # @!attribute [r] errors
66
+ # @return [Error, Array<Error>] An array of errors that are considered circuit
67
+ # failures. Default `[StandardError]`.
68
+ # @!attribute [r] exclude
69
+ # @return [Error, Array<Error>] An array of errors that will be captured and
70
+ # considered circuit failures. Default `[]`.
71
+ # @!attribute [r] cache
72
+ # @return [Cache::Interface] The cache backend. Default `Cache::Null.new`
73
+ # @!attribute [r] notifier
74
+ # @return [Events::Notifier] A Faulty notifier. Default `Events::Notifier.new`
75
+ # @!attribute [r] storage
76
+ # @return [Storage::Interface] The storage backend. Default `Storage::Memory.new`
77
+ Options = Struct.new(
78
+ :cache_expires_in,
79
+ :cache_refreshes_after,
80
+ :cache_refresh_jitter,
81
+ :cool_down,
82
+ :evaluation_window,
83
+ :rate_threshold,
84
+ :sample_threshold,
85
+ :errors,
86
+ :exclude,
87
+ :cache,
88
+ :notifier,
89
+ :storage
90
+ ) do
91
+ include ImmutableOptions
92
+
93
+ private
94
+
95
+ def defaults
96
+ {
97
+ cache_expires_in: 86_400,
98
+ cache_refreshes_after: 900,
99
+ cool_down: 300,
100
+ errors: [StandardError],
101
+ exclude: [],
102
+ evaluation_window: 60,
103
+ rate_threshold: 0.5,
104
+ sample_threshold: 3
105
+ }
106
+ end
107
+
108
+ def required
109
+ %i[
110
+ cache
111
+ cool_down
112
+ errors
113
+ exclude
114
+ evaluation_window
115
+ rate_threshold
116
+ sample_threshold
117
+ notifier
118
+ storage
119
+ ]
120
+ end
121
+
122
+ def finalize
123
+ self.cache ||= Cache::Default.new
124
+ self.notifier ||= Events::Notifier.new
125
+ self.storage ||= Storage::Memory.new
126
+ self.errors = [errors] if errors && !errors.is_a?(Array)
127
+ self.exclude = [exclude] if exclude && !exclude.is_a?(Array)
128
+
129
+ unless cache_refreshes_after.nil?
130
+ self.cache_refresh_jitter = 0.2 * cache_refreshes_after
131
+ end
132
+ end
133
+ end
134
+
135
+ # @param name [String] The name of the circuit
136
+ # @param options [Hash] Attributes for {Options}
137
+ # @yield [Options] For setting options in a block
138
+ def initialize(name, **options, &block)
139
+ raise ArgumentError, 'name must be a String' unless name.is_a?(String)
140
+
141
+ @name = name
142
+ @options = Options.new(options, &block)
143
+ end
144
+
145
+ # Run the circuit as with {#run}, but return a {Result}
146
+ #
147
+ # This is syntax sugar for running a circuit and rescuing an error
148
+ #
149
+ # @example
150
+ # result = Faulty.circuit(:api).try_run do
151
+ # api.get
152
+ # end
153
+ #
154
+ # response = if result.ok?
155
+ # result.get
156
+ # else
157
+ # { error: result.error.message }
158
+ # end
159
+ #
160
+ # @example
161
+ # # The Result object has a fetch method that can return a default value
162
+ # # if an error occurs
163
+ # result = Faulty.circuit(:api).try_run do
164
+ # api.get
165
+ # end.fetch({})
166
+ #
167
+ # @param (see #run)
168
+ # @yield (see #run)
169
+ # @raise If the block raises an error not in the error list, or if the error
170
+ # is excluded.
171
+ # @return [Result<Object, Error>] A result where the ok value is the return
172
+ # value of the block, or the error value is an error captured by the
173
+ # circuit.
174
+ def try_run(**options, &block)
175
+ Result.new(ok: run(**options, &block))
176
+ rescue FaultyError => e
177
+ Result.new(error: e)
178
+ end
179
+
180
+ # Run a block protected by this circuit
181
+ #
182
+ # If the circuit is closed, the block will run. Any exceptions raised inside
183
+ # the block will be checked against the error and exclude options to determine
184
+ # whether that error should be captured. If the error is captured, this
185
+ # run will be recorded as a failure.
186
+ #
187
+ # If the circuit exceeds the failure conditions, this circuit will be tripped
188
+ # and marked as open. Any future calls to run will not execute the block, but
189
+ # instead wait for the cool down period. Once the cool down period passes,
190
+ # the circuit transitions to half-open, and the block will be allowed to run.
191
+ #
192
+ # If the circuit fails again while half-open, the circuit will be closed for
193
+ # a second cool down period. However, if the circuit completes successfully,
194
+ # the circuit will be closed and reset to its initial state.
195
+ #
196
+ # @param cache [String, nil] A cache key, or nil if caching is not desired
197
+ # @yield The block to protect with this circuit
198
+ # @raise If the block raises an error not in the error list, or if the error
199
+ # is excluded.
200
+ # @raise {OpenCircuitError} if the circuit is open
201
+ # @raise {CircuitTrippedError} if this run causes the circuit to trip. It's
202
+ # possible for concurrent runs to simultaneously trip the circuit if the
203
+ # storage engine is not concurrency-safe.
204
+ # @raise {CircuitFailureError} if this run fails, but doesn't cause the
205
+ # circuit to trip
206
+ # @return The return value of the block
207
+ def run(cache: nil, &block)
208
+ cached_value = cache_read(cache)
209
+ # return cached unless cached.nil?
210
+ return cached_value if !cached_value.nil? && !cache_should_refresh?(cache)
211
+ return run_skipped(cached_value) unless status.can_run?
212
+
213
+ run_exec(cached_value, cache, &block)
214
+ end
215
+
216
+ # Force the circuit to stay open until unlocked
217
+ #
218
+ # @return [self]
219
+ def lock_open!
220
+ storage.lock(self, :open)
221
+ self
222
+ end
223
+
224
+ # Force the circuit to stay closed until unlocked
225
+ #
226
+ # @return [self]
227
+ def lock_closed!
228
+ storage.lock(self, :closed)
229
+ self
230
+ end
231
+
232
+ # Remove any open or closed locks
233
+ #
234
+ # @return [self]
235
+ def unlock!
236
+ storage.unlock(self)
237
+ self
238
+ end
239
+
240
+ # Reset this circuit to its initial state
241
+ #
242
+ # This removes the current state, all history, and locks
243
+ #
244
+ # @return [self]
245
+ def reset!
246
+ storage.reset(self)
247
+ self
248
+ end
249
+
250
+ # Get the current status of the circuit
251
+ #
252
+ # This method is not safe for concurrent operations, so it's unsafe
253
+ # to check this method and make runtime decisions based on that. However,
254
+ # it's useful for getting a non-synchronized snapshot of a circuit.
255
+ #
256
+ # @return [Status]
257
+ def status
258
+ storage.status(self)
259
+ end
260
+
261
+ # Get the history of runs of this circuit
262
+ #
263
+ # The history is an array of tuples where the first value is
264
+ # the run time, and the second value is a boolean which is true
265
+ # if the run was successful.
266
+ #
267
+ # @return [Array<Array>>] An array of tuples of [run_time, is_success]
268
+ def history
269
+ storage.history(self)
270
+ end
271
+
272
+ private
273
+
274
+ # Process a skipped run
275
+ #
276
+ # @param cached_value The cached value if one is available
277
+ # @return The result from cache if available
278
+ def run_skipped(cached_value)
279
+ skipped!
280
+ raise OpenCircuitError.new(nil, self) if cached_value.nil?
281
+
282
+ cached_value
283
+ end
284
+
285
+ # Excecute a run
286
+ #
287
+ # @param cached_value The cached value if one is available
288
+ # @param cache_key [String, nil] The cache key if one is given
289
+ # @return The run result
290
+ def run_exec(cached_value, cache_key)
291
+ result = yield
292
+ success!
293
+ cache_write(cache_key, result)
294
+ result
295
+ rescue *options.errors => e
296
+ raise if options.exclude.any? { |ex| e.is_a?(ex) }
297
+
298
+ if cached_value.nil?
299
+ raise CircuitTrippedError.new(nil, self) if failure!(e)
300
+
301
+ raise CircuitFailureError.new(nil, self)
302
+ else
303
+ cached_value
304
+ end
305
+ end
306
+
307
+ # @return [Boolean] True if the circuit transitioned to closed
308
+ def success!
309
+ status = storage.entry(self, Faulty.current_time, true)
310
+ closed = false
311
+ closed = close! if should_close?(status)
312
+
313
+ options.notifier.notify(:circuit_success, circuit: self, status: status)
314
+ closed
315
+ end
316
+
317
+ # @return [Boolean] True if the circuit transitioned to open
318
+ def failure!(error)
319
+ status = storage.entry(self, Faulty.current_time, false)
320
+ options.notifier.notify(:circuit_failure, circuit: self, status: status, error: error)
321
+
322
+ opened = if status.half_open?
323
+ reopen!(error, status.opened_at)
324
+ elsif status.fails_threshold?
325
+ open!(error)
326
+ else
327
+ false
328
+ end
329
+
330
+ opened
331
+ end
332
+
333
+ def skipped!
334
+ options.notifier.notify(:circuit_skipped, circuit: self)
335
+ end
336
+
337
+ # @return [Boolean] True if the circuit transitioned from closed to open
338
+ def open!(error)
339
+ opened = storage.open(self, Faulty.current_time)
340
+ options.notifier.notify(:circuit_opened, circuit: self, error: error) if opened
341
+ opened
342
+ end
343
+
344
+ # @return [Boolean] True if the circuit was reopened
345
+ def reopen!(error, previous_opened_at)
346
+ reopened = storage.reopen(self, Faulty.current_time, previous_opened_at)
347
+ options.notifier.notify(:circuit_reopened, circuit: self, error: error) if reopened
348
+ reopened
349
+ end
350
+
351
+ # @return [Boolean] True if the circuit transitioned from half-open to closed
352
+ def close!
353
+ closed = storage.close(self)
354
+ options.notifier.notify(:circuit_closed, circuit: self) if closed
355
+ closed
356
+ end
357
+
358
+ # Test whether we should close after a successful run
359
+ #
360
+ # Currently this is always true if the circuit is half-open, which is the
361
+ # traditional behavior for a circuit-breaker
362
+ #
363
+ # @return [Boolean] True if we should close the circuit from half-open
364
+ def should_close?(status)
365
+ status.half_open?
366
+ end
367
+
368
+ # Read from the cache if it is configured
369
+ #
370
+ # @param key The key to read from the cache
371
+ # @return The cached value, or nil if not present
372
+ def cache_read(key)
373
+ return unless key
374
+
375
+ result = options.cache.read(key.to_s)
376
+ event = result.nil? ? :circuit_cache_miss : :circuit_cache_hit
377
+ options.notifier.notify(event, circuit: self, key: key)
378
+ result
379
+ end
380
+
381
+ # Write to the cache if it is configured
382
+ #
383
+ # @param key The key to read from the cache
384
+ # @return [void]
385
+ def cache_write(key, value)
386
+ return unless key
387
+
388
+ options.notifier.notify(:circuit_cache_write, circuit: self, key: key)
389
+ options.cache.write(key.to_s, value, expires_in: options.cache_expires_in)
390
+
391
+ unless options.cache_refreshes_after.nil?
392
+ options.cache.write(cache_refresh_key(key.to_s), next_refresh_time, expires_in: options.cache_expires_in)
393
+ end
394
+ end
395
+
396
+ # Check whether the cache should be refreshed
397
+ #
398
+ # Should be called only if cache is present
399
+ #
400
+ # @return [Boolean] true if the cache should be refreshed
401
+ def cache_should_refresh?(key)
402
+ time = options.cache.read(cache_refresh_key(key.to_s)).to_i
403
+ time + (rand * 2 - 1) * options.cache_refresh_jitter < Faulty.current_time
404
+ end
405
+
406
+ # Get the next time to refresh the cache when writing to it
407
+ #
408
+ # @return [Integer] The timestamp to refresh at
409
+ def next_refresh_time
410
+ (Faulty.current_time + options.cache_refreshes_after).floor
411
+ end
412
+
413
+ # Get the corresponding cache refresh key for a given cache key
414
+ #
415
+ # We use this to force a cache entry to refresh before it has expired
416
+ #
417
+ # @return [String] The cache refresh key
418
+ def cache_refresh_key(key)
419
+ "#{key}#{CACHE_REFRESH_SUFFIX}"
420
+ end
421
+
422
+ # Get a random number from 0.0 to 1.0 for use with cache jitter
423
+ #
424
+ # @return [Float] A random number from 0.0 to 1.0
425
+ def rand
426
+ SecureRandom.random_number
427
+ end
428
+
429
+ # Alias to the storage engine from options
430
+ #
431
+ # @return [Storage::Interface]
432
+ def storage
433
+ options.storage
434
+ end
435
+ end
436
+ end