faulty 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.rubocop.yml +85 -0
- data/.travis.yml +44 -0
- data/.yardopts +3 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +20 -0
- data/README.md +559 -0
- data/bin/check-version +10 -0
- data/bin/console +12 -0
- data/bin/rspec +29 -0
- data/bin/rubocop +29 -0
- data/bin/yard +29 -0
- data/bin/yardoc +29 -0
- data/bin/yri +29 -0
- data/faulty.gemspec +43 -0
- data/lib/faulty.rb +118 -0
- data/lib/faulty/cache.rb +13 -0
- data/lib/faulty/cache/default.rb +48 -0
- data/lib/faulty/cache/fault_tolerant_proxy.rb +74 -0
- data/lib/faulty/cache/interface.rb +44 -0
- data/lib/faulty/cache/mock.rb +39 -0
- data/lib/faulty/cache/null.rb +23 -0
- data/lib/faulty/cache/rails.rb +37 -0
- data/lib/faulty/circuit.rb +436 -0
- data/lib/faulty/error.rb +66 -0
- data/lib/faulty/events.rb +25 -0
- data/lib/faulty/events/callback_listener.rb +42 -0
- data/lib/faulty/events/listener_interface.rb +18 -0
- data/lib/faulty/events/log_listener.rb +88 -0
- data/lib/faulty/events/notifier.rb +25 -0
- data/lib/faulty/immutable_options.rb +40 -0
- data/lib/faulty/result.rb +150 -0
- data/lib/faulty/scope.rb +117 -0
- data/lib/faulty/status.rb +165 -0
- data/lib/faulty/storage.rb +11 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +178 -0
- data/lib/faulty/storage/interface.rb +161 -0
- data/lib/faulty/storage/memory.rb +195 -0
- data/lib/faulty/storage/redis.rb +335 -0
- data/lib/faulty/version.rb +8 -0
- metadata +306 -0
data/lib/faulty/scope.rb
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
# A {Scope} is a group of options and circuits
|
5
|
+
#
|
6
|
+
# For most use-cases the default scope should be used, however, it's possible
|
7
|
+
# to create any number of scopes for applications that require a more complex
|
8
|
+
# configuration or for testing.
|
9
|
+
#
|
10
|
+
# For the most part, scopes are independent, however for some cache and
|
11
|
+
# storage backends, you will need to ensure that the cache keys and circuit
|
12
|
+
# names don't overlap between scopes. For example, if using the Redis storage
|
13
|
+
# backend, you should specify different key prefixes for each scope.
|
14
|
+
class Scope
|
15
|
+
attr_reader :options
|
16
|
+
|
17
|
+
# Options for {Scope}
|
18
|
+
#
|
19
|
+
# @!attribute [r] cache
|
20
|
+
# @return [Cache::Interface] A cache backend if you want
|
21
|
+
# to use Faulty's cache support. Automatically wrapped in a
|
22
|
+
# {Cache::FaultTolerantProxy}. Default `Cache::Default.new`.
|
23
|
+
# @!attribute [r] storage
|
24
|
+
# @return [Storage::Interface] The storage backend.
|
25
|
+
# Automatically wrapped in a {Storage::FaultTolerantProxy}.
|
26
|
+
# Default `Storage::Memory.new`.
|
27
|
+
# @!attribute [r] listeners
|
28
|
+
# @return [Array] listeners Faulty event listeners
|
29
|
+
# @!attribute [r] notifier
|
30
|
+
# @return [Events::Notifier] A Faulty notifier. If given, listeners are
|
31
|
+
# ignored.
|
32
|
+
Options = Struct.new(
|
33
|
+
:cache,
|
34
|
+
:storage,
|
35
|
+
:listeners,
|
36
|
+
:notifier
|
37
|
+
) do
|
38
|
+
include ImmutableOptions
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def finalize
|
43
|
+
self.notifier ||= Events::Notifier.new(listeners || [])
|
44
|
+
|
45
|
+
self.storage ||= Storage::Memory.new
|
46
|
+
unless storage.fault_tolerant?
|
47
|
+
self.storage = Storage::FaultTolerantProxy.new(storage, notifier: notifier)
|
48
|
+
end
|
49
|
+
|
50
|
+
self.cache ||= Cache::Default.new
|
51
|
+
unless cache.fault_tolerant?
|
52
|
+
self.cache = Cache::FaultTolerantProxy.new(cache, notifier: notifier)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def required
|
57
|
+
%i[cache storage notifier]
|
58
|
+
end
|
59
|
+
|
60
|
+
def defaults
|
61
|
+
{
|
62
|
+
listeners: [Events::LogListener.new]
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# Create a new Faulty Scope
|
68
|
+
#
|
69
|
+
# Note, the process of creating a new scope is not thread safe,
|
70
|
+
# so make sure scopes are setup before spawning threads.
|
71
|
+
#
|
72
|
+
# @see Options
|
73
|
+
# @param options [Hash] Attributes for {Options}
|
74
|
+
# @yield [Options] For setting options in a block
|
75
|
+
def initialize(**options, &block)
|
76
|
+
@circuits = Concurrent::Map.new
|
77
|
+
@options = Options.new(options, &block)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Create or retrieve a circuit
|
81
|
+
#
|
82
|
+
# Within a scope, circuit instances have unique names, so if the given circuit
|
83
|
+
# name already exists, then the existing circuit will be returned, otherwise
|
84
|
+
# a new circuit will be created. If an existing circuit is returned, then
|
85
|
+
# the {options} param and block are ignored.
|
86
|
+
#
|
87
|
+
# @param name [String] The name of the circuit
|
88
|
+
# @param options [Hash] Attributes for {Circuit::Options}
|
89
|
+
# @yield [Circuit::Options] For setting options in a block
|
90
|
+
# @return [Circuit] The new circuit or the existing circuit if it already exists
|
91
|
+
def circuit(name, **options, &block)
|
92
|
+
name = name.to_s
|
93
|
+
options = options.merge(circuit_options)
|
94
|
+
@circuits.compute_if_absent(name) do
|
95
|
+
Circuit.new(name, **options, &block)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# Get a list of all circuit names
|
100
|
+
#
|
101
|
+
# @return [Array<String>] The circuit names
|
102
|
+
def list_circuits
|
103
|
+
options.storage.list
|
104
|
+
end
|
105
|
+
|
106
|
+
private
|
107
|
+
|
108
|
+
# Get circuit options from the scope options
|
109
|
+
#
|
110
|
+
# @return [Hash] The circuit options
|
111
|
+
def circuit_options
|
112
|
+
options = @options.to_h
|
113
|
+
options.delete(:listeners)
|
114
|
+
options
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
# The status of a circuit
|
5
|
+
#
|
6
|
+
# Includes information like the state and locks. Also calculates
|
7
|
+
# whether a circuit can be run, or if it has failed a threshold.
|
8
|
+
#
|
9
|
+
# @!attribute [r] state
|
10
|
+
# @return [:open, :closed] The stored circuit state. This is always open
|
11
|
+
# or closed. Half-open is calculated from the current time. For that
|
12
|
+
# reason, calling state directly should be avoided. Instead use the
|
13
|
+
# status methods {#open?}, {#closed?}, and {#half_open?}.
|
14
|
+
# Default `:closed`
|
15
|
+
# @!attribute [r] lock
|
16
|
+
# @return [:open, :closed, nil] If the circuit is locked, the state that
|
17
|
+
# it is locked in. Default `nil`.
|
18
|
+
# @!attribute [r] opened_at
|
19
|
+
# @return [Integer, nil] If the circuit is open, the timestamp that it was
|
20
|
+
# opened. This is not necessarily reset when the circuit is closed.
|
21
|
+
# Default `nil`.
|
22
|
+
# @!attribute [r] failure_rate
|
23
|
+
# @return [Float] A number from 0 to 1 representing the percentage of
|
24
|
+
# failures for the circuit. For exmaple 0.5 represents a 50% failure rate.
|
25
|
+
# @!attribute [r] sample_size
|
26
|
+
# @return [Integer] The number of samples used to calculate the failure rate.
|
27
|
+
# @!attribute [r] options
|
28
|
+
# @return [Circuit::Options] The options for the circuit
|
29
|
+
# @!attribute [r] stub
|
30
|
+
# @return [Boolean] True if this status is a stub and not calculated from
|
31
|
+
# the storage backend. Used by {Storage::FaultTolerantProxy} when
|
32
|
+
# returning the status for an offline storage backend. Default `false`.
|
33
|
+
Status = Struct.new(
|
34
|
+
:state,
|
35
|
+
:lock,
|
36
|
+
:opened_at,
|
37
|
+
:failure_rate,
|
38
|
+
:sample_size,
|
39
|
+
:options,
|
40
|
+
:stub
|
41
|
+
) do
|
42
|
+
include ImmutableOptions
|
43
|
+
|
44
|
+
# The allowed state values
|
45
|
+
STATES = %i[
|
46
|
+
open
|
47
|
+
closed
|
48
|
+
].freeze
|
49
|
+
|
50
|
+
# The allowed lock values
|
51
|
+
LOCKS = %i[
|
52
|
+
open
|
53
|
+
closed
|
54
|
+
].freeze
|
55
|
+
|
56
|
+
# Create a new `Status` from a list of circuit runs
|
57
|
+
#
|
58
|
+
# For storage backends that store entries, this automatically calculates
|
59
|
+
# failure_rate and sample size.
|
60
|
+
#
|
61
|
+
# @param entries [Array<Array>] An array of entry tuples. See
|
62
|
+
# {Circuit#history} for details
|
63
|
+
# @param hash [Hash] The status attributes minus failure_rate and
|
64
|
+
# sample_size
|
65
|
+
# @return [Status]
|
66
|
+
def self.from_entries(entries, **hash)
|
67
|
+
failures = 0
|
68
|
+
sample_size = 0
|
69
|
+
entries.each do |(time, success)|
|
70
|
+
next unless time > Faulty.current_time - hash[:options].evaluation_window
|
71
|
+
|
72
|
+
sample_size += 1
|
73
|
+
failures += 1 unless success
|
74
|
+
end
|
75
|
+
|
76
|
+
new(hash.merge(
|
77
|
+
sample_size: sample_size,
|
78
|
+
failure_rate: sample_size.zero? ? 0.0 : failures.to_f / sample_size
|
79
|
+
))
|
80
|
+
end
|
81
|
+
|
82
|
+
# Whether the circuit is open
|
83
|
+
#
|
84
|
+
# This is mutually exclusive with {#closed?} and {#half_open?}
|
85
|
+
#
|
86
|
+
# @return [Boolean] True if open
|
87
|
+
def open?
|
88
|
+
state == :open && opened_at + options.cool_down > Faulty.current_time
|
89
|
+
end
|
90
|
+
|
91
|
+
# Whether the circuit is closed
|
92
|
+
#
|
93
|
+
# This is mutually exclusive with {#open?} and {#half_open?}
|
94
|
+
#
|
95
|
+
# @return [Boolean] True if closed
|
96
|
+
def closed?
|
97
|
+
state == :closed
|
98
|
+
end
|
99
|
+
|
100
|
+
# Whether the circuit is half-open
|
101
|
+
#
|
102
|
+
# This is mutually exclusive with {#open?} and {#closed?}
|
103
|
+
#
|
104
|
+
# @return [Boolean] True if half-open
|
105
|
+
def half_open?
|
106
|
+
state == :open && opened_at + options.cool_down <= Faulty.current_time
|
107
|
+
end
|
108
|
+
|
109
|
+
# Whether the circuit is locked open
|
110
|
+
#
|
111
|
+
# @return [Boolean] True if locked open
|
112
|
+
def locked_open?
|
113
|
+
lock == :open
|
114
|
+
end
|
115
|
+
|
116
|
+
# Whether the circuit is locked closed
|
117
|
+
#
|
118
|
+
# @return [Boolean] True if locked closed
|
119
|
+
def locked_closed?
|
120
|
+
lock == :closed
|
121
|
+
end
|
122
|
+
|
123
|
+
# Whether the circuit can be run
|
124
|
+
#
|
125
|
+
# Takes the circuit state, locks and cooldown into account
|
126
|
+
#
|
127
|
+
# @return [Boolean] True if the circuit can be run
|
128
|
+
def can_run?
|
129
|
+
return false if locked_open?
|
130
|
+
|
131
|
+
closed? || locked_closed? || half_open?
|
132
|
+
end
|
133
|
+
|
134
|
+
# Whether the circuit fails the sample size and rate thresholds
|
135
|
+
#
|
136
|
+
# @return [Boolean] True if the circuit fails the thresholds
|
137
|
+
def fails_threshold?
|
138
|
+
return false if sample_size < options.sample_threshold
|
139
|
+
|
140
|
+
failure_rate >= options.rate_threshold
|
141
|
+
end
|
142
|
+
|
143
|
+
private
|
144
|
+
|
145
|
+
def finalize
|
146
|
+
raise ArgumentError, "state must be a symbol in #{self.class}::STATES" unless STATES.include?(state)
|
147
|
+
unless lock.nil? || LOCKS.include?(state)
|
148
|
+
raise ArgumentError, "lock must be a symbol in #{self.class}::LOCKS or nil"
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def required
|
153
|
+
%i[state failure_rate sample_size options stub]
|
154
|
+
end
|
155
|
+
|
156
|
+
def defaults
|
157
|
+
{
|
158
|
+
state: :closed,
|
159
|
+
failure_rate: 0.0,
|
160
|
+
sample_size: 0,
|
161
|
+
stub: false
|
162
|
+
}
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
module Storage
|
5
|
+
# A wrapper for storage backends that may raise errors
|
6
|
+
#
|
7
|
+
# {Scope} automatically wraps all non-fault-tolerant storage backends with
|
8
|
+
# this class.
|
9
|
+
#
|
10
|
+
# If the storage backend raises a `StandardError`, it will be captured and
|
11
|
+
# sent to the notifier.
|
12
|
+
class FaultTolerantProxy
|
13
|
+
attr_reader :options
|
14
|
+
|
15
|
+
# Options for {FaultTolerantProxy}
|
16
|
+
#
|
17
|
+
# @!attribute [r] notifier
|
18
|
+
# @return [Events::Notifier] A Faulty notifier
|
19
|
+
Options = Struct.new(
|
20
|
+
:notifier
|
21
|
+
) do
|
22
|
+
include ImmutableOptions
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def required
|
27
|
+
%i[notifier]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param storage [Storage::Interface] The storage backend to wrap
|
32
|
+
# @param options [Hash] Attributes for {Options}
|
33
|
+
# @yield [Options] For setting options in a block
|
34
|
+
def initialize(storage, **options, &block)
|
35
|
+
@storage = storage
|
36
|
+
@options = Options.new(options, &block)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Add a history entry safely
|
40
|
+
#
|
41
|
+
# @see Interface#entry
|
42
|
+
# @param (see Interface#entry)
|
43
|
+
# @return (see Interface#entry)
|
44
|
+
def entry(circuit, time, success)
|
45
|
+
@storage.entry(circuit, time, success)
|
46
|
+
rescue StandardError => e
|
47
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :entry, error: e)
|
48
|
+
stub_status(circuit)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Safely mark a circuit as open
|
52
|
+
#
|
53
|
+
# @see Interface#open
|
54
|
+
# @param (see Interface#open)
|
55
|
+
# @return (see Interface#open)
|
56
|
+
def open(circuit)
|
57
|
+
@storage.open(circuit)
|
58
|
+
rescue StandardError => e
|
59
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :open, error: e)
|
60
|
+
false
|
61
|
+
end
|
62
|
+
|
63
|
+
# Safely mark a circuit as reopened
|
64
|
+
#
|
65
|
+
# @see Interface#reopen
|
66
|
+
# @param (see Interface#reopen)
|
67
|
+
# @return (see Interface#reopen)
|
68
|
+
def reopen(circuit)
|
69
|
+
@storage.reopen(circuit)
|
70
|
+
rescue StandardError => e
|
71
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :reopen, error: e)
|
72
|
+
false
|
73
|
+
end
|
74
|
+
|
75
|
+
# Safely mark a circuit as closed
|
76
|
+
#
|
77
|
+
# @see Interface#close
|
78
|
+
# @param (see Interface#close)
|
79
|
+
# @return (see Interface#close)
|
80
|
+
def close(circuit)
|
81
|
+
@storage.close(circuit)
|
82
|
+
rescue StandardError => e
|
83
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :close, error: e)
|
84
|
+
false
|
85
|
+
end
|
86
|
+
|
87
|
+
# Since lock is not called in normal operation, it does not capture
|
88
|
+
# errors
|
89
|
+
#
|
90
|
+
# @see Interface#lock
|
91
|
+
# @param (see Interface#lock)
|
92
|
+
# @return (see Interface#lock)
|
93
|
+
def lock(circuit, state)
|
94
|
+
@storage.lock(circuit, state)
|
95
|
+
end
|
96
|
+
|
97
|
+
# Since unlock is not called in normal operation, it does not capture
|
98
|
+
# errors
|
99
|
+
#
|
100
|
+
# @see Interface#unlock
|
101
|
+
# @param (see Interface#unlock)
|
102
|
+
# @return (see Interface#unlock)
|
103
|
+
def unlock(circuit)
|
104
|
+
@storage.unlock(circuit)
|
105
|
+
end
|
106
|
+
|
107
|
+
# Since reset is not called in normal operation, it does not capture
|
108
|
+
# errors
|
109
|
+
#
|
110
|
+
# @see Interface#reset
|
111
|
+
# @param (see Interface#reset)
|
112
|
+
# @return (see Interface#reset)
|
113
|
+
def reset(circuit)
|
114
|
+
@storage.reset(circuit)
|
115
|
+
end
|
116
|
+
|
117
|
+
# Safely get the status of a circuit
|
118
|
+
#
|
119
|
+
# If the backend is unavailable, this returns a stub status that
|
120
|
+
# indicates that the circuit is closed.
|
121
|
+
#
|
122
|
+
# @see Interface#status
|
123
|
+
# @param (see Interface#status)
|
124
|
+
# @return (see Interface#status)
|
125
|
+
def status(circuit)
|
126
|
+
@storage.status(circuit)
|
127
|
+
rescue StandardError => e
|
128
|
+
options.notifier.notify(:storage_failure, circuit: circuit, action: :status, error: e)
|
129
|
+
stub_status(circuit)
|
130
|
+
end
|
131
|
+
|
132
|
+
# Since history is not called in normal operation, it does not capture
|
133
|
+
# errors
|
134
|
+
#
|
135
|
+
# @see Interface#history
|
136
|
+
# @param (see Interface#history)
|
137
|
+
# @return (see Interface#history)
|
138
|
+
def history(circuit)
|
139
|
+
@storage.history(circuit)
|
140
|
+
end
|
141
|
+
|
142
|
+
# Safely get the list of circuit names
|
143
|
+
#
|
144
|
+
# If the backend is unavailable, this returns an empty array
|
145
|
+
#
|
146
|
+
# @see Interface#list
|
147
|
+
# @param (see Interface#list)
|
148
|
+
# @return (see Interface#list)
|
149
|
+
def list
|
150
|
+
@storage.list
|
151
|
+
rescue StandardError => e
|
152
|
+
options.notifier.notify(:storage_failure, action: :list, error: e)
|
153
|
+
[]
|
154
|
+
end
|
155
|
+
|
156
|
+
# This cache makes any storage fault tolerant, so this is always `true`
|
157
|
+
#
|
158
|
+
# @return [true]
|
159
|
+
def fault_tolerant?
|
160
|
+
true
|
161
|
+
end
|
162
|
+
|
163
|
+
private
|
164
|
+
|
165
|
+
# Create a stub status object to close the circuit by default
|
166
|
+
#
|
167
|
+
# @return [Status] The stub status
|
168
|
+
def stub_status(circuit)
|
169
|
+
Faulty::Status.new(
|
170
|
+
cool_down: circuit.options.cool_down,
|
171
|
+
stub: true,
|
172
|
+
sample_threshold: circuit.options.sample_threshold,
|
173
|
+
rate_threshold: circuit.options.rate_threshold
|
174
|
+
)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|