faulty 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.rubocop.yml +85 -0
- data/.travis.yml +44 -0
- data/.yardopts +3 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +20 -0
- data/README.md +559 -0
- data/bin/check-version +10 -0
- data/bin/console +12 -0
- data/bin/rspec +29 -0
- data/bin/rubocop +29 -0
- data/bin/yard +29 -0
- data/bin/yardoc +29 -0
- data/bin/yri +29 -0
- data/faulty.gemspec +43 -0
- data/lib/faulty.rb +118 -0
- data/lib/faulty/cache.rb +13 -0
- data/lib/faulty/cache/default.rb +48 -0
- data/lib/faulty/cache/fault_tolerant_proxy.rb +74 -0
- data/lib/faulty/cache/interface.rb +44 -0
- data/lib/faulty/cache/mock.rb +39 -0
- data/lib/faulty/cache/null.rb +23 -0
- data/lib/faulty/cache/rails.rb +37 -0
- data/lib/faulty/circuit.rb +436 -0
- data/lib/faulty/error.rb +66 -0
- data/lib/faulty/events.rb +25 -0
- data/lib/faulty/events/callback_listener.rb +42 -0
- data/lib/faulty/events/listener_interface.rb +18 -0
- data/lib/faulty/events/log_listener.rb +88 -0
- data/lib/faulty/events/notifier.rb +25 -0
- data/lib/faulty/immutable_options.rb +40 -0
- data/lib/faulty/result.rb +150 -0
- data/lib/faulty/scope.rb +117 -0
- data/lib/faulty/status.rb +165 -0
- data/lib/faulty/storage.rb +11 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +178 -0
- data/lib/faulty/storage/interface.rb +161 -0
- data/lib/faulty/storage/memory.rb +195 -0
- data/lib/faulty/storage/redis.rb +335 -0
- data/lib/faulty/version.rb +8 -0
- metadata +306 -0
@@ -0,0 +1,161 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
module Storage
|
5
|
+
# The interface required for a storage backend implementation
|
6
|
+
#
|
7
|
+
# This is for documentation only and is not loaded
|
8
|
+
class Interface
|
9
|
+
# Add a circuit run entry to storage
|
10
|
+
#
|
11
|
+
# The backend may choose to store this in whatever manner it chooses as
|
12
|
+
# long as it can implement the other read methods.
|
13
|
+
#
|
14
|
+
# @param circuit [Circuit] The circuit that ran
|
15
|
+
# @param time [Integer] The unix timestamp for the run
|
16
|
+
# @param success [Boolean] True if the run succeeded
|
17
|
+
# @return [Status] The circuit status after the run is added
|
18
|
+
def entry(circuit, time, success)
|
19
|
+
raise NotImplementedError
|
20
|
+
end
|
21
|
+
|
22
|
+
# Set the circuit state to open
|
23
|
+
#
|
24
|
+
# If multiple parallel processes open the circuit simultaneously, open
|
25
|
+
# may be called more than once. If so, this method should return true
|
26
|
+
# only once, when the circuit transitions from closed to open.
|
27
|
+
#
|
28
|
+
# If the backend does not support locking or atomic operations, then
|
29
|
+
# it may always return true, but that could result in duplicate open
|
30
|
+
# notifications.
|
31
|
+
#
|
32
|
+
# If returning true, this method also updates opened_at to the
|
33
|
+
# current time.
|
34
|
+
#
|
35
|
+
# @param circuit [Circuit] The circuit to open
|
36
|
+
# @param opened_at [Integer] The timestmp the circuit was opened at
|
37
|
+
# @return [Boolean] True if the circuit transitioned from closed to open
|
38
|
+
def open(circuit, opened_at)
|
39
|
+
raise NotImplementedError
|
40
|
+
end
|
41
|
+
|
42
|
+
# Reset the opened_at time for a half_open circuit
|
43
|
+
#
|
44
|
+
# If multiple parallel processes open the circuit simultaneously, reopen
|
45
|
+
# may be called more than once. If so, this method should return true
|
46
|
+
# only once, when the circuit updates the opened_at value. It can use the
|
47
|
+
# value from previous_opened_at to do a compare-and-set operation.
|
48
|
+
#
|
49
|
+
# If the backend does not support locking or atomic operations, then
|
50
|
+
# it may always return true, but that could result in duplicate reopen
|
51
|
+
# notifications.
|
52
|
+
#
|
53
|
+
# @param circuit [Circuit] The circuit to reopen
|
54
|
+
# @param opened_at [Integer] The timestmp the circuit was opened at
|
55
|
+
# @param previous_opened_at [Integer] The last known value of opened_at.
|
56
|
+
# Can be used to comare-and-set.
|
57
|
+
# @return [Boolean] True if the opened_at time was updated
|
58
|
+
def reopen(circuit, opened_at, previous_opened_at)
|
59
|
+
raise NotImplementedError
|
60
|
+
end
|
61
|
+
|
62
|
+
# Set the circuit state to closed
|
63
|
+
#
|
64
|
+
# If multiple parallel processes close the circuit simultaneously, close
|
65
|
+
# may be called more than once. If so, this method should return true
|
66
|
+
# only once, when the circuit transitions from open to closed.
|
67
|
+
#
|
68
|
+
# If the backend does not support locking or atomic operations, then
|
69
|
+
# it may always return true, but that could result in duplicate close
|
70
|
+
# notifications.
|
71
|
+
#
|
72
|
+
# @return [Boolean] True if the circuit transitioned from open to closed
|
73
|
+
def close(circuit)
|
74
|
+
raise NotImplementedError
|
75
|
+
end
|
76
|
+
|
77
|
+
# Lock the circuit in a given state
|
78
|
+
#
|
79
|
+
# No concurrency gurantees are provided for locking
|
80
|
+
#
|
81
|
+
# @param circuit [Circuit] The circuit to lock
|
82
|
+
# @param state [:open, :closed] The state to lock the circuit in
|
83
|
+
# @return [void]
|
84
|
+
def lock(circuit, state)
|
85
|
+
raise NotImplementedError
|
86
|
+
end
|
87
|
+
|
88
|
+
# Unlock the circuit from any state
|
89
|
+
#
|
90
|
+
# No concurrency gurantees are provided for locking
|
91
|
+
#
|
92
|
+
# @param circuit [Circuit] The circuit to unlock
|
93
|
+
# @return [void]
|
94
|
+
def unlock(circuit)
|
95
|
+
raise NotImplementedError
|
96
|
+
end
|
97
|
+
|
98
|
+
# Reset the circuit to a fresh state
|
99
|
+
#
|
100
|
+
# Clears all circuit status including entries, state, locks,
|
101
|
+
# opened_at, and any other values that would affect Status.
|
102
|
+
#
|
103
|
+
# No concurrency gurantees are provided for resetting
|
104
|
+
#
|
105
|
+
# @param circuit [Circuit] The circuit to unlock
|
106
|
+
# @return [void]
|
107
|
+
def reset(circuit)
|
108
|
+
raise NotImplementedError
|
109
|
+
end
|
110
|
+
|
111
|
+
# Get the status object for a circuit
|
112
|
+
#
|
113
|
+
# No concurrency gurantees are provided for getting status. It's possible
|
114
|
+
# that status may represent a circuit in the middle of modification.
|
115
|
+
#
|
116
|
+
# @param circuit [Circuit] The circuit to get status for
|
117
|
+
# @return [Status] The current status
|
118
|
+
def status(circuit)
|
119
|
+
raise NotImplementedError
|
120
|
+
end
|
121
|
+
|
122
|
+
# Get the entry history of a circuit
|
123
|
+
#
|
124
|
+
# No concurrency gurantees are provided for getting status. It's possible
|
125
|
+
# that status may represent a circuit in the middle of modification.
|
126
|
+
#
|
127
|
+
# A storage backend may choose not to implement this method and instead
|
128
|
+
# return an empty array.
|
129
|
+
#
|
130
|
+
# Each item in the history array is an array of two items (a tuple) of
|
131
|
+
# `[run_time, succeeded]`, where `run_time` is a unix timestamp, and
|
132
|
+
# `succeeded` is a boolean, true if the run succeeded.
|
133
|
+
#
|
134
|
+
# @param circuit [Circuit] The circuit to get history for
|
135
|
+
# @return [Array<Array>] An array of history tuples
|
136
|
+
def history(circuit)
|
137
|
+
raise NotImplementedError
|
138
|
+
end
|
139
|
+
|
140
|
+
# Get a list of all circuit names
|
141
|
+
#
|
142
|
+
# If the storage backend does not support listing circuits, this may
|
143
|
+
# return an empty array.
|
144
|
+
#
|
145
|
+
# @return [Array<String>]
|
146
|
+
def list
|
147
|
+
raise NotImplementedError
|
148
|
+
end
|
149
|
+
|
150
|
+
# Can this storage backend raise an error?
|
151
|
+
#
|
152
|
+
# If the storage backend returns false from this method, it will be wrapped
|
153
|
+
# in a {FaultTolerantProxy}, otherwise it will be used as-is.
|
154
|
+
#
|
155
|
+
# @return [Boolean] True if this cache backend is fault tolerant
|
156
|
+
def fault_tolerant?
|
157
|
+
raise NotImplementedError
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
module Storage
|
5
|
+
# The default in-memory storage for circuits
|
6
|
+
#
|
7
|
+
# This implementation is most suitable to single-process, low volume
|
8
|
+
# usage. It is thread-safe and circuit state is shared across threads.
|
9
|
+
#
|
10
|
+
# Circuit state and runs are stored in memory. Although runs have a maximum
|
11
|
+
# size within a circuit, there is no limit on the number of circuits that
|
12
|
+
# can be stored. This means the user should be careful about the number of
|
13
|
+
# circuits that are created. To that end, it's a good idea to avoid
|
14
|
+
# dynamically-named circuits with this backend.
|
15
|
+
#
|
16
|
+
# For a more robust distributed implementation, use the {Redis} storage
|
17
|
+
# backend.
|
18
|
+
#
|
19
|
+
# This can be used as a reference implementation for storage backends that
|
20
|
+
# store a list of circuit run entries.
|
21
|
+
class Memory
|
22
|
+
attr_reader :options
|
23
|
+
|
24
|
+
# Options for {Memory}
|
25
|
+
#
|
26
|
+
# @!attribute [r] max_sample_size
|
27
|
+
# @return [Integer] The number of cache run entries to keep in memory
|
28
|
+
# for each circuit. Default `100`.
|
29
|
+
Options = Struct.new(:max_sample_size) do
|
30
|
+
include ImmutableOptions
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def defaults
|
35
|
+
{ max_sample_size: 100 }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# The internal object for storing a circuit
|
40
|
+
#
|
41
|
+
# @private
|
42
|
+
MemoryCircuit = Struct.new(:state, :runs, :opened_at, :lock) do
|
43
|
+
def initialize
|
44
|
+
self.state = Concurrent::Atom.new(:closed)
|
45
|
+
self.runs = Concurrent::MVar.new([], dup_on_deref: true)
|
46
|
+
self.opened_at = Concurrent::Atom.new(nil)
|
47
|
+
self.lock = nil
|
48
|
+
end
|
49
|
+
|
50
|
+
# Create a status object from the current circuit state
|
51
|
+
#
|
52
|
+
# @param circuit_options [Circuit::Options] The circuit options object
|
53
|
+
# @return [Status] The newly created status
|
54
|
+
def status(circuit_options)
|
55
|
+
status = nil
|
56
|
+
runs.borrow do |locked_runs|
|
57
|
+
status = Faulty::Status.from_entries(
|
58
|
+
locked_runs,
|
59
|
+
state: state.value,
|
60
|
+
lock: lock,
|
61
|
+
opened_at: opened_at.value,
|
62
|
+
options: circuit_options
|
63
|
+
)
|
64
|
+
end
|
65
|
+
|
66
|
+
status
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# @param options [Hash] Attributes for {Options}
|
71
|
+
# @yield [Options] For setting options in a block
|
72
|
+
def initialize(**options, &block)
|
73
|
+
@circuits = Concurrent::Map.new
|
74
|
+
@options = Options.new(options, &block)
|
75
|
+
end
|
76
|
+
|
77
|
+
# Add an entry to storage
|
78
|
+
#
|
79
|
+
# @see Interface#entry
|
80
|
+
# @param (see Interface#entry)
|
81
|
+
# @return (see Interface#entry)
|
82
|
+
def entry(circuit, time, success)
|
83
|
+
memory = fetch(circuit)
|
84
|
+
memory.runs.borrow do |runs|
|
85
|
+
runs.push([time, success])
|
86
|
+
runs.pop if runs.size > options.max_sample_size
|
87
|
+
end
|
88
|
+
memory.status(circuit.options)
|
89
|
+
end
|
90
|
+
|
91
|
+
# Mark a circuit as open
|
92
|
+
#
|
93
|
+
# @see Interface#open
|
94
|
+
# @param (see Interface#open)
|
95
|
+
# @return (see Interface#open)
|
96
|
+
def open(circuit, opened_at)
|
97
|
+
memory = fetch(circuit)
|
98
|
+
opened = memory.state.compare_and_set(:closed, :open)
|
99
|
+
memory.opened_at.reset(opened_at) if opened
|
100
|
+
opened
|
101
|
+
end
|
102
|
+
|
103
|
+
# Mark a circuit as reopened
|
104
|
+
#
|
105
|
+
# @see Interface#reopen
|
106
|
+
# @param (see Interface#reopen)
|
107
|
+
# @return (see Interface#reopen)
|
108
|
+
def reopen(circuit, opened_at, previous_opened_at)
|
109
|
+
memory = fetch(circuit)
|
110
|
+
memory.opened_at.compare_and_set(previous_opened_at, opened_at)
|
111
|
+
end
|
112
|
+
|
113
|
+
# Mark a circuit as closed
|
114
|
+
#
|
115
|
+
# @see Interface#close
|
116
|
+
# @param (see Interface#close)
|
117
|
+
# @return (see Interface#close)
|
118
|
+
def close(circuit)
|
119
|
+
memory = fetch(circuit)
|
120
|
+
memory.runs.modify { |_old| [] }
|
121
|
+
memory.state.compare_and_set(:open, :closed)
|
122
|
+
end
|
123
|
+
|
124
|
+
# Lock a circuit open or closed
|
125
|
+
#
|
126
|
+
# @see Interface#lock
|
127
|
+
# @param (see Interface#lock)
|
128
|
+
# @return (see Interface#lock)
|
129
|
+
def lock(circuit, state)
|
130
|
+
memory = fetch(circuit)
|
131
|
+
memory.lock = state
|
132
|
+
end
|
133
|
+
|
134
|
+
# Unlock a circuit
|
135
|
+
#
|
136
|
+
# @see Interface#unlock
|
137
|
+
# @param (see Interface#unlock)
|
138
|
+
# @return (see Interface#unlock)
|
139
|
+
def unlock(circuit)
|
140
|
+
memory = fetch(circuit)
|
141
|
+
memory.lock = nil
|
142
|
+
end
|
143
|
+
|
144
|
+
# Reset a circuit
|
145
|
+
#
|
146
|
+
# @see Interface#reset
|
147
|
+
# @param (see Interface#reset)
|
148
|
+
# @return (see Interface#reset)
|
149
|
+
def reset(circuit)
|
150
|
+
@circuits.delete(circuit.name)
|
151
|
+
end
|
152
|
+
|
153
|
+
# Get the status of a circuit
|
154
|
+
#
|
155
|
+
# @see Interface#status
|
156
|
+
# @param (see Interface#status)
|
157
|
+
# @return (see Interface#status)
|
158
|
+
def status(circuit)
|
159
|
+
fetch(circuit).status(circuit.options)
|
160
|
+
end
|
161
|
+
|
162
|
+
# Get the circuit history up to `max_sample_size`
|
163
|
+
#
|
164
|
+
# @see Interface#history
|
165
|
+
# @param (see Interface#history)
|
166
|
+
# @return (see Interface#history)
|
167
|
+
def history(circuit)
|
168
|
+
fetch(circuit).runs.value
|
169
|
+
end
|
170
|
+
|
171
|
+
# Get a list of circuit names
|
172
|
+
#
|
173
|
+
# @return [Array<String>] The circuit names
|
174
|
+
def list
|
175
|
+
@circuits.keys
|
176
|
+
end
|
177
|
+
|
178
|
+
# Memory storage is fault-tolerant by default
|
179
|
+
#
|
180
|
+
# @return [true]
|
181
|
+
def fault_tolerant?
|
182
|
+
true
|
183
|
+
end
|
184
|
+
|
185
|
+
private
|
186
|
+
|
187
|
+
# Fetch circuit storage safely or create it if it doesn't exist
|
188
|
+
#
|
189
|
+
# @return [MemoryCircuit]
|
190
|
+
def fetch(circuit)
|
191
|
+
@circuits.compute_if_absent(circuit.name) { MemoryCircuit.new }
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,335 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Faulty
|
4
|
+
module Storage
|
5
|
+
class Redis # rubocop:disable Metrics/ClassLength
|
6
|
+
# Separates the time/status for history entry strings
|
7
|
+
ENTRY_SEPARATOR = ':'
|
8
|
+
|
9
|
+
attr_reader :options
|
10
|
+
|
11
|
+
# Options for {Redis}
|
12
|
+
#
|
13
|
+
# @!attribute [r] client
|
14
|
+
# @return [Redis,ConnectionPool] The Redis instance or a ConnectionPool
|
15
|
+
# used to connect to Redis. Default `::Redis.new`
|
16
|
+
# @!attribute [r] key_prefix
|
17
|
+
# @return [String] A string prepended to all Redis keys used to store
|
18
|
+
# circuit state. Default `faulty`.
|
19
|
+
# @!attribute [r] key_separator
|
20
|
+
# @return [String] A string used to separate the parts of the Redis keys
|
21
|
+
# used to store circuit state. Defaulty `:`.
|
22
|
+
# @!attribute [r] max_sample_size
|
23
|
+
# @return [Integer] The number of cache run entries to keep in memory
|
24
|
+
# for each circuit. Default `100`.
|
25
|
+
# @!attribute [r] sample_ttl
|
26
|
+
# @return [Integer] The maximum number of seconds to store a
|
27
|
+
# circuit run history entry. Default `100`.
|
28
|
+
# @!attribute [r] circuit_ttl
|
29
|
+
# @return [Integer] The maximum number of seconds to keep a circuit.
|
30
|
+
# A value of `nil` disables circuit expiration.
|
31
|
+
# Default `604_800` (1 week).
|
32
|
+
# @!attribute [r] list_granularity
|
33
|
+
# @return [Integer] The number of seconds after which a new set is
|
34
|
+
# created to store circuit names. The old set is kept until
|
35
|
+
# circuit_ttl expires. Default `3600` (1 hour).
|
36
|
+
Options = Struct.new(
|
37
|
+
:client,
|
38
|
+
:key_prefix,
|
39
|
+
:key_separator,
|
40
|
+
:max_sample_size,
|
41
|
+
:sample_ttl,
|
42
|
+
:circuit_ttl,
|
43
|
+
:list_granularity
|
44
|
+
) do
|
45
|
+
include ImmutableOptions
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def defaults
|
50
|
+
{
|
51
|
+
key_prefix: 'faulty',
|
52
|
+
key_separator: ':',
|
53
|
+
max_sample_size: 100,
|
54
|
+
sample_ttl: 1800,
|
55
|
+
circuit_ttl: 604_800,
|
56
|
+
list_granularity: 3600
|
57
|
+
}
|
58
|
+
end
|
59
|
+
|
60
|
+
def required
|
61
|
+
%i[list_granularity]
|
62
|
+
end
|
63
|
+
|
64
|
+
def finalize
|
65
|
+
self.client = ::Redis.new unless client
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# @param options [Hash] Attributes for {Options}
|
70
|
+
# @yield [Options] For setting options in a block
|
71
|
+
def initialize(**options, &block)
|
72
|
+
@options = Options.new(options, &block)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Add an entry to storage
|
76
|
+
#
|
77
|
+
# @see Interface#entry
|
78
|
+
# @param (see Interface#entry)
|
79
|
+
# @return (see Interface#entry)
|
80
|
+
def entry(circuit, time, success)
|
81
|
+
key = entries_key(circuit)
|
82
|
+
pipe do |r|
|
83
|
+
r.sadd(list_key, circuit.name)
|
84
|
+
r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
|
85
|
+
r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
|
86
|
+
r.ltrim(key, 0, options.max_sample_size - 1)
|
87
|
+
r.expire(key, options.sample_ttl) if options.sample_ttl
|
88
|
+
end
|
89
|
+
|
90
|
+
status(circuit)
|
91
|
+
end
|
92
|
+
|
93
|
+
# Mark a circuit as open
|
94
|
+
#
|
95
|
+
# @see Interface#open
|
96
|
+
# @param (see Interface#open)
|
97
|
+
# @return (see Interface#open)
|
98
|
+
def open(circuit, opened_at)
|
99
|
+
redis do |r|
|
100
|
+
opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
|
101
|
+
r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
|
102
|
+
opened
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# Mark a circuit as reopened
|
107
|
+
#
|
108
|
+
# @see Interface#reopen
|
109
|
+
# @param (see Interface#reopen)
|
110
|
+
# @return (see Interface#reopen)
|
111
|
+
def reopen(circuit, opened_at, previous_opened_at)
|
112
|
+
redis do |r|
|
113
|
+
compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
# Mark a circuit as closed
|
118
|
+
#
|
119
|
+
# @see Interface#close
|
120
|
+
# @param (see Interface#close)
|
121
|
+
# @return (see Interface#close)
|
122
|
+
def close(circuit)
|
123
|
+
redis do |r|
|
124
|
+
closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
|
125
|
+
r.del(entries_key(circuit)) if closed
|
126
|
+
closed
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# Lock a circuit open or closed
|
131
|
+
#
|
132
|
+
# The circuit_ttl does not apply to locks
|
133
|
+
#
|
134
|
+
# @see Interface#lock
|
135
|
+
# @param (see Interface#lock)
|
136
|
+
# @return (see Interface#lock)
|
137
|
+
def lock(circuit, state)
|
138
|
+
redis { |r| r.set(lock_key(circuit), state) }
|
139
|
+
end
|
140
|
+
|
141
|
+
# Unlock a circuit
|
142
|
+
#
|
143
|
+
# @see Interface#unlock
|
144
|
+
# @param (see Interface#unlock)
|
145
|
+
# @return (see Interface#unlock)
|
146
|
+
def unlock(circuit)
|
147
|
+
redis { |r| r.del(lock_key(circuit)) }
|
148
|
+
end
|
149
|
+
|
150
|
+
# Reset a circuit
|
151
|
+
#
|
152
|
+
# @see Interface#reset
|
153
|
+
# @param (see Interface#reset)
|
154
|
+
# @return (see Interface#reset)
|
155
|
+
def reset(circuit)
|
156
|
+
pipe do |r|
|
157
|
+
r.del(
|
158
|
+
entries_key(circuit),
|
159
|
+
opened_at_key(circuit),
|
160
|
+
lock_key(circuit)
|
161
|
+
)
|
162
|
+
r.set(state_key(circuit), 'closed', ex: options.circuit_ttl)
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
# Get the status of a circuit
|
167
|
+
#
|
168
|
+
# @see Interface#status
|
169
|
+
# @param (see Interface#status)
|
170
|
+
# @return (see Interface#status)
|
171
|
+
def status(circuit)
|
172
|
+
futures = {}
|
173
|
+
pipe do |r|
|
174
|
+
futures[:state] = r.get(state_key(circuit))
|
175
|
+
futures[:lock] = r.get(lock_key(circuit))
|
176
|
+
futures[:opened_at] = r.get(opened_at_key(circuit))
|
177
|
+
futures[:entries] = r.lrange(entries_key(circuit), 0, -1)
|
178
|
+
end
|
179
|
+
|
180
|
+
Faulty::Status.from_entries(
|
181
|
+
map_entries(futures[:entries].value),
|
182
|
+
state: futures[:state].value&.to_sym || :closed,
|
183
|
+
lock: futures[:lock].value&.to_sym,
|
184
|
+
opened_at: futures[:opened_at].value ? futures[:opened_at].value.to_i : nil,
|
185
|
+
options: circuit.options
|
186
|
+
)
|
187
|
+
end
|
188
|
+
|
189
|
+
# Get the circuit history up to `max_sample_size`
|
190
|
+
#
|
191
|
+
# @see Interface#history
|
192
|
+
# @param (see Interface#history)
|
193
|
+
# @return (see Interface#history)
|
194
|
+
def history(circuit)
|
195
|
+
entries = redis { |r| r.lrange(entries_key(circuit), 0, -1) }
|
196
|
+
map_entries(entries).reverse
|
197
|
+
end
|
198
|
+
|
199
|
+
def list
|
200
|
+
redis { |r| r.sunion(*all_list_keys) }
|
201
|
+
end
|
202
|
+
|
203
|
+
# Redis storage is not fault-tolerant
|
204
|
+
#
|
205
|
+
# @return [true]
|
206
|
+
def fault_tolerant?
|
207
|
+
false
|
208
|
+
end
|
209
|
+
|
210
|
+
private
|
211
|
+
|
212
|
+
# Generate a key from its parts
|
213
|
+
#
|
214
|
+
# @return [String] The key
|
215
|
+
def key(*parts)
|
216
|
+
[options.key_prefix, *parts].join(options.key_separator)
|
217
|
+
end
|
218
|
+
|
219
|
+
def ckey(circuit, *parts)
|
220
|
+
key('circuit', circuit.name, *parts)
|
221
|
+
end
|
222
|
+
|
223
|
+
# @return [String] The key for circuit state
|
224
|
+
def state_key(circuit)
|
225
|
+
ckey(circuit, 'state')
|
226
|
+
end
|
227
|
+
|
228
|
+
# @return [String] The key for circuit run history entries
|
229
|
+
def entries_key(circuit)
|
230
|
+
ckey(circuit, 'entries')
|
231
|
+
end
|
232
|
+
|
233
|
+
# @return [String] The key for circuit locks
|
234
|
+
def lock_key(circuit)
|
235
|
+
ckey(circuit, 'lock')
|
236
|
+
end
|
237
|
+
|
238
|
+
# @return [String] The key for circuit opened_at
|
239
|
+
def opened_at_key(circuit)
|
240
|
+
ckey(circuit, 'opened_at')
|
241
|
+
end
|
242
|
+
|
243
|
+
# Get the current key to add circuit names to
|
244
|
+
def list_key
|
245
|
+
key('list', current_list_block)
|
246
|
+
end
|
247
|
+
|
248
|
+
# Get all active circuit list keys
|
249
|
+
#
|
250
|
+
# We use a rolling list of redis sets to store circuit names. This way we
|
251
|
+
# can maintain this index, while still using Redis to expire old circuits.
|
252
|
+
# Whenever we add a circuit to the list, we add it to the current set. A
|
253
|
+
# new set is created every `options.list_granularity` seconds.
|
254
|
+
#
|
255
|
+
# When reading the list, we union all sets together, which gets us the
|
256
|
+
# full list.
|
257
|
+
#
|
258
|
+
# Each set has its own expiration, so that the oldest sets will
|
259
|
+
# automatically be deleted from Redis after `options.circuit_ttl`.
|
260
|
+
#
|
261
|
+
# It is possible for a single circuit name to be a part of many of these
|
262
|
+
# sets. This is the space trade-off we make in exchange for automatic
|
263
|
+
# expiration.
|
264
|
+
#
|
265
|
+
# @return [Array<String>] An array of redis keys for circuit name sets
|
266
|
+
def all_list_keys
|
267
|
+
num_blocks = (options.circuit_ttl.to_f / options.list_granularity).floor + 1
|
268
|
+
start_block = current_list_block - num_blocks + 1
|
269
|
+
num_blocks.times.map do |i|
|
270
|
+
key('list', start_block + i)
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
# Get the block number for the current list set
|
275
|
+
#
|
276
|
+
# @return [Integer] The current block number
|
277
|
+
def current_list_block
|
278
|
+
(Faulty.current_time.to_f / options.list_granularity).floor
|
279
|
+
end
|
280
|
+
|
281
|
+
# Set a value in Redis only if it matches a list of current values
|
282
|
+
#
|
283
|
+
# @param redis [Redis] The redis connection
|
284
|
+
# @param key [String] The redis key to CAS
|
285
|
+
# @param old [Array<String>] A list of previous values that pass the
|
286
|
+
# comparison
|
287
|
+
# @param new [String] The new value to set if the compare passes
|
288
|
+
# @return [Boolean] True if the value was set to `new`, false if the CAS
|
289
|
+
# failed
|
290
|
+
def compare_and_set(redis, key, old, new)
|
291
|
+
result = redis.watch(key) do
|
292
|
+
if old.include?(redis.get(key))
|
293
|
+
redis.multi { |m| m.set(key, new) }
|
294
|
+
else
|
295
|
+
redis.unwatch
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
result[0] == 'OK'
|
300
|
+
end
|
301
|
+
|
302
|
+
# Yield a Redis connection
|
303
|
+
#
|
304
|
+
# @yield [Redis] Yields the connection to the block
|
305
|
+
# @return The value returned from the block
|
306
|
+
def redis
|
307
|
+
if options.client.respond_to?(:with)
|
308
|
+
options.client.with { |redis| yield redis }
|
309
|
+
else
|
310
|
+
yield options.client
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
314
|
+
# Yield a pipelined Redis connection
|
315
|
+
#
|
316
|
+
# @yield [Redis::Pipeline] Yields the connection to the block
|
317
|
+
# @return [void]
|
318
|
+
def pipe
|
319
|
+
redis { |r| r.pipelined { |p| yield p } }
|
320
|
+
end
|
321
|
+
|
322
|
+
# Map raw Redis history entries to Faulty format
|
323
|
+
#
|
324
|
+
# @see Storage::Interface
|
325
|
+
# @param raw_entries [Array<String>] The raw Redis entries
|
326
|
+
# @return [Array<Array>] The Faulty-formatted entries
|
327
|
+
def map_entries(raw_entries)
|
328
|
+
raw_entries.map do |e|
|
329
|
+
time, state = e.split(ENTRY_SEPARATOR)
|
330
|
+
[time.to_i, state == '1']
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
end
|