faulty 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.rubocop.yml +85 -0
- data/.travis.yml +44 -0
- data/.yardopts +3 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +20 -0
- data/README.md +559 -0
- data/bin/check-version +10 -0
- data/bin/console +12 -0
- data/bin/rspec +29 -0
- data/bin/rubocop +29 -0
- data/bin/yard +29 -0
- data/bin/yardoc +29 -0
- data/bin/yri +29 -0
- data/faulty.gemspec +43 -0
- data/lib/faulty.rb +118 -0
- data/lib/faulty/cache.rb +13 -0
- data/lib/faulty/cache/default.rb +48 -0
- data/lib/faulty/cache/fault_tolerant_proxy.rb +74 -0
- data/lib/faulty/cache/interface.rb +44 -0
- data/lib/faulty/cache/mock.rb +39 -0
- data/lib/faulty/cache/null.rb +23 -0
- data/lib/faulty/cache/rails.rb +37 -0
- data/lib/faulty/circuit.rb +436 -0
- data/lib/faulty/error.rb +66 -0
- data/lib/faulty/events.rb +25 -0
- data/lib/faulty/events/callback_listener.rb +42 -0
- data/lib/faulty/events/listener_interface.rb +18 -0
- data/lib/faulty/events/log_listener.rb +88 -0
- data/lib/faulty/events/notifier.rb +25 -0
- data/lib/faulty/immutable_options.rb +40 -0
- data/lib/faulty/result.rb +150 -0
- data/lib/faulty/scope.rb +117 -0
- data/lib/faulty/status.rb +165 -0
- data/lib/faulty/storage.rb +11 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +178 -0
- data/lib/faulty/storage/interface.rb +161 -0
- data/lib/faulty/storage/memory.rb +195 -0
- data/lib/faulty/storage/redis.rb +335 -0
- data/lib/faulty/version.rb +8 -0
- metadata +306 -0
data/lib/faulty/scope.rb
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
# A {Scope} is a group of options and circuits
|
5
|
+
#
|
6
|
+
# For most use-cases the default scope should be used, however, it's possible
|
7
|
+
# to create any number of scopes for applications that require a more complex
|
8
|
+
# configuration or for testing.
|
9
|
+
#
|
10
|
+
# For the most part, scopes are independent, however for some cache and
|
11
|
+
# storage backends, you will need to ensure that the cache keys and circuit
|
12
|
+
# names don't overlap between scopes. For example, if using the Redis storage
|
13
|
+
# backend, you should specify different key prefixes for each scope.
|
14
|
+
class Scope
|
15
|
+
attr_reader :options
|
16
|
+
|
17
|
+
# Options for {Scope}
|
18
|
+
#
|
19
|
+
# @!attribute [r] cache
|
20
|
+
# @return [Cache::Interface] A cache backend if you want
|
21
|
+
# to use Faulty's cache support. Automatically wrapped in a
|
22
|
+
# {Cache::FaultTolerantProxy}. Default `Cache::Default.new`.
|
23
|
+
# @!attribute [r] storage
|
24
|
+
# @return [Storage::Interface] The storage backend.
|
25
|
+
# Automatically wrapped in a {Storage::FaultTolerantProxy}.
|
26
|
+
# Default `Storage::Memory.new`.
|
27
|
+
# @!attribute [r] listeners
|
28
|
+
# @return [Array] listeners Faulty event listeners
|
29
|
+
# @!attribute [r] notifier
|
30
|
+
# @return [Events::Notifier] A Faulty notifier. If given, listeners are
|
31
|
+
# ignored.
|
32
|
+
Options = Struct.new(
|
33
|
+
:cache,
|
34
|
+
:storage,
|
35
|
+
:listeners,
|
36
|
+
:notifier
|
37
|
+
) do
|
38
|
+
include ImmutableOptions
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def finalize
|
43
|
+
self.notifier ||= Events::Notifier.new(listeners || [])
|
44
|
+
|
45
|
+
self.storage ||= Storage::Memory.new
|
46
|
+
unless storage.fault_tolerant?
|
47
|
+
self.storage = Storage::FaultTolerantProxy.new(storage, notifier: notifier)
|
48
|
+
end
|
49
|
+
|
50
|
+
self.cache ||= Cache::Default.new
|
51
|
+
unless cache.fault_tolerant?
|
52
|
+
self.cache = Cache::FaultTolerantProxy.new(cache, notifier: notifier)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def required
|
57
|
+
%i[cache storage notifier]
|
58
|
+
end
|
59
|
+
|
60
|
+
def defaults
|
61
|
+
{
|
62
|
+
listeners: [Events::LogListener.new]
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# Create a new Faulty Scope
|
68
|
+
#
|
69
|
+
# Note, the process of creating a new scope is not thread safe,
|
70
|
+
# so make sure scopes are setup before spawning threads.
|
71
|
+
#
|
72
|
+
# @see Options
|
73
|
+
# @param options [Hash] Attributes for {Options}
|
74
|
+
# @yield [Options] For setting options in a block
|
75
|
+
def initialize(**options, &block)
|
76
|
+
@circuits = Concurrent::Map.new
|
77
|
+
@options = Options.new(options, &block)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Create or retrieve a circuit
|
81
|
+
#
|
82
|
+
# Within a scope, circuit instances have unique names, so if the given circuit
|
83
|
+
# name already exists, then the existing circuit will be returned, otherwise
|
84
|
+
# a new circuit will be created. If an existing circuit is returned, then
|
85
|
+
# the {options} param and block are ignored.
|
86
|
+
#
|
87
|
+
# @param name [String] The name of the circuit
|
88
|
+
# @param options [Hash] Attributes for {Circuit::Options}
|
89
|
+
# @yield [Circuit::Options] For setting options in a block
|
90
|
+
# @return [Circuit] The new circuit or the existing circuit if it already exists
|
91
|
+
def circuit(name, **options, &block)
|
92
|
+
name = name.to_s
|
93
|
+
options = options.merge(circuit_options)
|
94
|
+
@circuits.compute_if_absent(name) do
|
95
|
+
Circuit.new(name, **options, &block)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# Get a list of all circuit names
|
100
|
+
#
|
101
|
+
# @return [Array<String>] The circuit names
|
102
|
+
def list_circuits
|
103
|
+
options.storage.list
|
104
|
+
end
|
105
|
+
|
106
|
+
private
|
107
|
+
|
108
|
+
# Get circuit options from the scope options
|
109
|
+
#
|
110
|
+
# @return [Hash] The circuit options
|
111
|
+
def circuit_options
|
112
|
+
options = @options.to_h
|
113
|
+
options.delete(:listeners)
|
114
|
+
options
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
# The status of a circuit
|
5
|
+
#
|
6
|
+
# Includes information like the state and locks. Also calculates
|
7
|
+
# whether a circuit can be run, or if it has failed a threshold.
|
8
|
+
#
|
9
|
+
# @!attribute [r] state
|
10
|
+
# @return [:open, :closed] The stored circuit state. This is always open
|
11
|
+
# or closed. Half-open is calculated from the current time. For that
|
12
|
+
# reason, calling state directly should be avoided. Instead use the
|
13
|
+
# status methods {#open?}, {#closed?}, and {#half_open?}.
|
14
|
+
# Default `:closed`
|
15
|
+
# @!attribute [r] lock
|
16
|
+
# @return [:open, :closed, nil] If the circuit is locked, the state that
|
17
|
+
# it is locked in. Default `nil`.
|
18
|
+
# @!attribute [r] opened_at
|
19
|
+
# @return [Integer, nil] If the circuit is open, the timestamp that it was
|
20
|
+
# opened. This is not necessarily reset when the circuit is closed.
|
21
|
+
# Default `nil`.
|
22
|
+
# @!attribute [r] failure_rate
|
23
|
+
# @return [Float] A number from 0 to 1 representing the percentage of
|
24
|
+
# failures for the circuit. For exmaple 0.5 represents a 50% failure rate.
|
25
|
+
# @!attribute [r] sample_size
|
26
|
+
# @return [Integer] The number of samples used to calculate the failure rate.
|
27
|
+
# @!attribute [r] options
|
28
|
+
# @return [Circuit::Options] The options for the circuit
|
29
|
+
# @!attribute [r] stub
|
30
|
+
# @return [Boolean] True if this status is a stub and not calculated from
|
31
|
+
# the storage backend. Used by {Storage::FaultTolerantProxy} when
|
32
|
+
# returning the status for an offline storage backend. Default `false`.
|
33
|
+
Status = Struct.new(
|
34
|
+
:state,
|
35
|
+
:lock,
|
36
|
+
:opened_at,
|
37
|
+
:failure_rate,
|
38
|
+
:sample_size,
|
39
|
+
:options,
|
40
|
+
:stub
|
41
|
+
) do
|
42
|
+
include ImmutableOptions
|
43
|
+
|
44
|
+
# The allowed state values
|
45
|
+
STATES = %i[
|
46
|
+
open
|
47
|
+
closed
|
48
|
+
].freeze
|
49
|
+
|
50
|
+
# The allowed lock values
|
51
|
+
LOCKS = %i[
|
52
|
+
open
|
53
|
+
closed
|
54
|
+
].freeze
|
55
|
+
|
56
|
+
# Create a new `Status` from a list of circuit runs
|
57
|
+
#
|
58
|
+
# For storage backends that store entries, this automatically calculates
|
59
|
+
# failure_rate and sample size.
|
60
|
+
#
|
61
|
+
# @param entries [Array<Array>] An array of entry tuples. See
|
62
|
+
# {Circuit#history} for details
|
63
|
+
# @param hash [Hash] The status attributes minus failure_rate and
|
64
|
+
# sample_size
|
65
|
+
# @return [Status]
|
66
|
+
def self.from_entries(entries, **hash)
|
67
|
+
failures = 0
|
68
|
+
sample_size = 0
|
69
|
+
entries.each do |(time, success)|
|
70
|
+
next unless time > Faulty.current_time - hash[:options].evaluation_window
|
71
|
+
|
72
|
+
sample_size += 1
|
73
|
+
failures += 1 unless success
|
74
|
+
end
|
75
|
+
|
76
|
+
new(hash.merge(
|
77
|
+
sample_size: sample_size,
|
78
|
+
failure_rate: sample_size.zero? ? 0.0 : failures.to_f / sample_size
|
79
|
+
))
|
80
|
+
end
|
81
|
+
|
82
|
+
# Whether the circuit is open
|
83
|
+
#
|
84
|
+
# This is mutually exclusive with {#closed?} and {#half_open?}
|
85
|
+
#
|
86
|
+
# @return [Boolean] True if open
|
87
|
+
def open?
|
88
|
+
state == :open && opened_at + options.cool_down > Faulty.current_time
|
89
|
+
end
|
90
|
+
|
91
|
+
# Whether the circuit is closed
|
92
|
+
#
|
93
|
+
# This is mutually exclusive with {#open?} and {#half_open?}
|
94
|
+
#
|
95
|
+
# @return [Boolean] True if closed
|
96
|
+
def closed?
|
97
|
+
state == :closed
|
98
|
+
end
|
99
|
+
|
100
|
+
# Whether the circuit is half-open
|
101
|
+
#
|
102
|
+
# This is mutually exclusive with {#open?} and {#closed?}
|
103
|
+
#
|
104
|
+
# @return [Boolean] True if half-open
|
105
|
+
def half_open?
|
106
|
+
state == :open && opened_at + options.cool_down <= Faulty.current_time
|
107
|
+
end
|
108
|
+
|
109
|
+
# Whether the circuit is locked open
|
110
|
+
#
|
111
|
+
# @return [Boolean] True if locked open
|
112
|
+
def locked_open?
|
113
|
+
lock == :open
|
114
|
+
end
|
115
|
+
|
116
|
+
# Whether the circuit is locked closed
|
117
|
+
#
|
118
|
+
# @return [Boolean] True if locked closed
|
119
|
+
def locked_closed?
|
120
|
+
lock == :closed
|
121
|
+
end
|
122
|
+
|
123
|
+
# Whether the circuit can be run
|
124
|
+
#
|
125
|
+
# Takes the circuit state, locks and cooldown into account
|
126
|
+
#
|
127
|
+
# @return [Boolean] True if the circuit can be run
|
128
|
+
def can_run?
|
129
|
+
return false if locked_open?
|
130
|
+
|
131
|
+
closed? || locked_closed? || half_open?
|
132
|
+
end
|
133
|
+
|
134
|
+
# Whether the circuit fails the sample size and rate thresholds
|
135
|
+
#
|
136
|
+
# @return [Boolean] True if the circuit fails the thresholds
|
137
|
+
def fails_threshold?
|
138
|
+
return false if sample_size < options.sample_threshold
|
139
|
+
|
140
|
+
failure_rate >= options.rate_threshold
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def finalize
|
146
|
+
raise ArgumentError, "state must be a symbol in #{self.class}::STATES" unless STATES.include?(state)
|
147
|
+
unless lock.nil? || LOCKS.include?(state)
|
148
|
+
raise ArgumentError, "lock must be a symbol in #{self.class}::LOCKS or nil"
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def required
|
153
|
+
%i[state failure_rate sample_size options stub]
|
154
|
+
end
|
155
|
+
|
156
|
+
def defaults
|
157
|
+
{
|
158
|
+
state: :closed,
|
159
|
+
failure_rate: 0.0,
|
160
|
+
sample_size: 0,
|
161
|
+
stub: false
|
162
|
+
}
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
module Storage
|
5
|
+
# A wrapper for storage backends that may raise errors
|
6
|
+
#
|
7
|
+
# {Scope} automatically wraps all non-fault-tolerant storage backends with
|
8
|
+
# this class.
|
9
|
+
#
|
10
|
+
# If the storage backend raises a `StandardError`, it will be captured and
|
11
|
+
# sent to the notifier.
|
12
|
+
class FaultTolerantProxy
|
13
|
+
attr_reader :options
|
14
|
+
|
15
|
+
# Options for {FaultTolerantProxy}
|
16
|
+
#
|
17
|
+
# @!attribute [r] notifier
|
18
|
+
# @return [Events::Notifier] A Faulty notifier
|
19
|
+
Options = Struct.new(
|
20
|
+
:notifier
|
21
|
+
) do
|
22
|
+
include ImmutableOptions
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def required
|
27
|
+
%i[notifier]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param storage [Storage::Interface] The storage backend to wrap
|
32
|
+
# @param options [Hash] Attributes for {Options}
|
33
|
+
# @yield [Options] For setting options in a block
|
34
|
+
def initialize(storage, **options, &block)
|
35
|
+
@storage = storage
|
36
|
+
@options = Options.new(options, &block)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Add a history entry safely
|
40
|
+
#
|
41
|
+
# @see Interface#entry
|
42
|
+
# @param (see Interface#entry)
|
43
|
+
# @return (see Interface#entry)
|
44
|
+
def entry(circuit, time, success)
|
45
|
+
@storage.entry(circuit, time, success)
|
46
|
+
rescue StandardError => e
|
47
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e)
|
48
|
+
stub_status(circuit)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Safely mark a circuit as open
|
52
|
+
#
|
53
|
+
# @see Interface#open
|
54
|
+
# @param (see Interface#open)
|
55
|
+
# @return (see Interface#open)
|
56
|
+
def open(circuit)
|
57
|
+
@storage.open(circuit)
|
58
|
+
rescue StandardError => e
|
59
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :open, error: e)
|
60
|
+
false
|
61
|
+
end
|
62
|
+
|
63
|
+
# Safely mark a circuit as reopened
|
64
|
+
#
|
65
|
+
# @see Interface#reopen
|
66
|
+
# @param (see Interface#reopen)
|
67
|
+
# @return (see Interface#reopen)
|
68
|
+
def reopen(circuit)
|
69
|
+
@storage.reopen(circuit)
|
70
|
+
rescue StandardError => e
|
71
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :reopen, error: e)
|
72
|
+
false
|
73
|
+
end
|
74
|
+
|
75
|
+
# Safely mark a circuit as closed
|
76
|
+
#
|
77
|
+
# @see Interface#close
|
78
|
+
# @param (see Interface#close)
|
79
|
+
# @return (see Interface#close)
|
80
|
+
def close(circuit)
|
81
|
+
@storage.close(circuit)
|
82
|
+
rescue StandardError => e
|
83
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :close, error: e)
|
84
|
+
false
|
85
|
+
end
|
86
|
+
|
87
|
+
# Since lock is not called in normal operation, it does not capture
|
88
|
+
# errors
|
89
|
+
#
|
90
|
+
# @see Interface#lock
|
91
|
+
# @param (see Interface#lock)
|
92
|
+
# @return (see Interface#lock)
|
93
|
+
def lock(circuit, state)
|
94
|
+
@storage.lock(circuit, state)
|
95
|
+
end
|
96
|
+
|
97
|
+
# Since unlock is not called in normal operation, it does not capture
|
98
|
+
# errors
|
99
|
+
#
|
100
|
+
# @see Interface#unlock
|
101
|
+
# @param (see Interface#unlock)
|
102
|
+
# @return (see Interface#unlock)
|
103
|
+
def unlock(circuit)
|
104
|
+
@storage.unlock(circuit)
|
105
|
+
end
|
106
|
+
|
107
|
+
# Since reset is not called in normal operation, it does not capture
|
108
|
+
# errors
|
109
|
+
#
|
110
|
+
# @see Interface#reset
|
111
|
+
# @param (see Interface#reset)
|
112
|
+
# @return (see Interface#reset)
|
113
|
+
def reset(circuit)
|
114
|
+
@storage.reset(circuit)
|
115
|
+
end
|
116
|
+
|
117
|
+
# Safely get the status of a circuit
|
118
|
+
#
|
119
|
+
# If the backend is unavailable, this returns a stub status that
|
120
|
+
# indicates that the circuit is closed.
|
121
|
+
#
|
122
|
+
# @see Interface#status
|
123
|
+
# @param (see Interface#status)
|
124
|
+
# @return (see Interface#status)
|
125
|
+
def status(circuit)
|
126
|
+
@storage.status(circuit)
|
127
|
+
rescue StandardError => e
|
128
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :status, error: e)
|
129
|
+
stub_status(circuit)
|
130
|
+
end
|
131
|
+
|
132
|
+
# Since history is not called in normal operation, it does not capture
|
133
|
+
# errors
|
134
|
+
#
|
135
|
+
# @see Interface#history
|
136
|
+
# @param (see Interface#history)
|
137
|
+
# @return (see Interface#history)
|
138
|
+
def history(circuit)
|
139
|
+
@storage.history(circuit)
|
140
|
+
end
|
141
|
+
|
142
|
+
# Safely get the list of circuit names
|
143
|
+
#
|
144
|
+
# If the backend is unavailable, this returns an empty array
|
145
|
+
#
|
146
|
+
# @see Interface#list
|
147
|
+
# @param (see Interface#list)
|
148
|
+
# @return (see Interface#list)
|
149
|
+
def list
|
150
|
+
@storage.list
|
151
|
+
rescue StandardError => e
|
152
|
+
options.notifier.notify(:storage_failure, action: :list, error: e)
|
153
|
+
[]
|
154
|
+
end
|
155
|
+
|
156
|
+
# This cache makes any storage fault tolerant, so this is always `true`
|
157
|
+
#
|
158
|
+
# @return [true]
|
159
|
+
def fault_tolerant?
|
160
|
+
true
|
161
|
+
end
|
162
|
+
|
163
|
+
private
|
164
|
+
|
165
|
+
# Create a stub status object to close the circuit by default
|
166
|
+
#
|
167
|
+
# @return [Status] The stub status
|
168
|
+
def stub_status(circuit)
|
169
|
+
Faulty::Status.new(
|
170
|
+
cool_down: circuit.options.cool_down,
|
171
|
+
stub: true,
|
172
|
+
sample_threshold: circuit.options.sample_threshold,
|
173
|
+
rate_threshold: circuit.options.rate_threshold
|
174
|
+
)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|