faulty 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Storage
5
+ # The interface required for a storage backend implementation
6
+ #
7
+ # This is for documentation only and is not loaded
8
+ class Interface
9
+ # Add a circuit run entry to storage
10
+ #
11
+ # The backend may choose to store this in whatever manner it chooses as
12
+ # long as it can implement the other read methods.
13
+ #
14
+ # @param circuit [Circuit] The circuit that ran
15
+ # @param time [Integer] The unix timestamp for the run
16
+ # @param success [Boolean] True if the run succeeded
17
+ # @return [Status] The circuit status after the run is added
18
+ def entry(circuit, time, success)
19
+ raise NotImplementedError
20
+ end
21
+
22
+ # Set the circuit state to open
23
+ #
24
+ # If multiple parallel processes open the circuit simultaneously, open
25
+ # may be called more than once. If so, this method should return true
26
+ # only once, when the circuit transitions from closed to open.
27
+ #
28
+ # If the backend does not support locking or atomic operations, then
29
+ # it may always return true, but that could result in duplicate open
30
+ # notifications.
31
+ #
32
+ # If returning true, this method also updates opened_at to the
33
+ # current time.
34
+ #
35
+ # @param circuit [Circuit] The circuit to open
36
+ # @param opened_at [Integer] The timestmp the circuit was opened at
37
+ # @return [Boolean] True if the circuit transitioned from closed to open
38
+ def open(circuit, opened_at)
39
+ raise NotImplementedError
40
+ end
41
+
42
+ # Reset the opened_at time for a half_open circuit
43
+ #
44
+ # If multiple parallel processes open the circuit simultaneously, reopen
45
+ # may be called more than once. If so, this method should return true
46
+ # only once, when the circuit updates the opened_at value. It can use the
47
+ # value from previous_opened_at to do a compare-and-set operation.
48
+ #
49
+ # If the backend does not support locking or atomic operations, then
50
+ # it may always return true, but that could result in duplicate reopen
51
+ # notifications.
52
+ #
53
+ # @param circuit [Circuit] The circuit to reopen
54
+ # @param opened_at [Integer] The timestmp the circuit was opened at
55
+ # @param previous_opened_at [Integer] The last known value of opened_at.
56
+ # Can be used to comare-and-set.
57
+ # @return [Boolean] True if the opened_at time was updated
58
+ def reopen(circuit, opened_at, previous_opened_at)
59
+ raise NotImplementedError
60
+ end
61
+
62
+ # Set the circuit state to closed
63
+ #
64
+ # If multiple parallel processes close the circuit simultaneously, close
65
+ # may be called more than once. If so, this method should return true
66
+ # only once, when the circuit transitions from open to closed.
67
+ #
68
+ # If the backend does not support locking or atomic operations, then
69
+ # it may always return true, but that could result in duplicate close
70
+ # notifications.
71
+ #
72
+ # @return [Boolean] True if the circuit transitioned from open to closed
73
+ def close(circuit)
74
+ raise NotImplementedError
75
+ end
76
+
77
+ # Lock the circuit in a given state
78
+ #
79
+ # No concurrency gurantees are provided for locking
80
+ #
81
+ # @param circuit [Circuit] The circuit to lock
82
+ # @param state [:open, :closed] The state to lock the circuit in
83
+ # @return [void]
84
+ def lock(circuit, state)
85
+ raise NotImplementedError
86
+ end
87
+
88
+ # Unlock the circuit from any state
89
+ #
90
+ # No concurrency gurantees are provided for locking
91
+ #
92
+ # @param circuit [Circuit] The circuit to unlock
93
+ # @return [void]
94
+ def unlock(circuit)
95
+ raise NotImplementedError
96
+ end
97
+
98
+ # Reset the circuit to a fresh state
99
+ #
100
+ # Clears all circuit status including entries, state, locks,
101
+ # opened_at, and any other values that would affect Status.
102
+ #
103
+ # No concurrency gurantees are provided for resetting
104
+ #
105
+ # @param circuit [Circuit] The circuit to unlock
106
+ # @return [void]
107
+ def reset(circuit)
108
+ raise NotImplementedError
109
+ end
110
+
111
+ # Get the status object for a circuit
112
+ #
113
+ # No concurrency gurantees are provided for getting status. It's possible
114
+ # that status may represent a circuit in the middle of modification.
115
+ #
116
+ # @param circuit [Circuit] The circuit to get status for
117
+ # @return [Status] The current status
118
+ def status(circuit)
119
+ raise NotImplementedError
120
+ end
121
+
122
+ # Get the entry history of a circuit
123
+ #
124
+ # No concurrency gurantees are provided for getting status. It's possible
125
+ # that status may represent a circuit in the middle of modification.
126
+ #
127
+ # A storage backend may choose not to implement this method and instead
128
+ # return an empty array.
129
+ #
130
+ # Each item in the history array is an array of two items (a tuple) of
131
+ # `[run_time, succeeded]`, where `run_time` is a unix timestamp, and
132
+ # `succeeded` is a boolean, true if the run succeeded.
133
+ #
134
+ # @param circuit [Circuit] The circuit to get history for
135
+ # @return [Array<Array>] An array of history tuples
136
+ def history(circuit)
137
+ raise NotImplementedError
138
+ end
139
+
140
+ # Get a list of all circuit names
141
+ #
142
+ # If the storage backend does not support listing circuits, this may
143
+ # return an empty array.
144
+ #
145
+ # @return [Array<String>]
146
+ def list
147
+ raise NotImplementedError
148
+ end
149
+
150
+ # Can this storage backend raise an error?
151
+ #
152
+ # If the storage backend returns false from this method, it will be wrapped
153
+ # in a {FaultTolerantProxy}, otherwise it will be used as-is.
154
+ #
155
+ # @return [Boolean] True if this cache backend is fault tolerant
156
+ def fault_tolerant?
157
+ raise NotImplementedError
158
+ end
159
+ end
160
+ end
161
+ end
@@ -0,0 +1,195 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Storage
5
+ # The default in-memory storage for circuits
6
+ #
7
+ # This implementation is most suitable to single-process, low volume
8
+ # usage. It is thread-safe and circuit state is shared across threads.
9
+ #
10
+ # Circuit state and runs are stored in memory. Although runs have a maximum
11
+ # size within a circuit, there is no limit on the number of circuits that
12
+ # can be stored. This means the user should be careful about the number of
13
+ # circuits that are created. To that end, it's a good idea to avoid
14
+ # dynamically-named circuits with this backend.
15
+ #
16
+ # For a more robust distributed implementation, use the {Redis} storage
17
+ # backend.
18
+ #
19
+ # This can be used as a reference implementation for storage backends that
20
+ # store a list of circuit run entries.
21
+ class Memory
22
+ attr_reader :options
23
+
24
+ # Options for {Memory}
25
+ #
26
+ # @!attribute [r] max_sample_size
27
+ # @return [Integer] The number of cache run entries to keep in memory
28
+ # for each circuit. Default `100`.
29
+ Options = Struct.new(:max_sample_size) do
30
+ include ImmutableOptions
31
+
32
+ private
33
+
34
+ def defaults
35
+ { max_sample_size: 100 }
36
+ end
37
+ end
38
+
39
+ # The internal object for storing a circuit
40
+ #
41
+ # @private
42
+ MemoryCircuit = Struct.new(:state, :runs, :opened_at, :lock) do
43
+ def initialize
44
+ self.state = Concurrent::Atom.new(:closed)
45
+ self.runs = Concurrent::MVar.new([], dup_on_deref: true)
46
+ self.opened_at = Concurrent::Atom.new(nil)
47
+ self.lock = nil
48
+ end
49
+
50
+ # Create a status object from the current circuit state
51
+ #
52
+ # @param circuit_options [Circuit::Options] The circuit options object
53
+ # @return [Status] The newly created status
54
+ def status(circuit_options)
55
+ status = nil
56
+ runs.borrow do |locked_runs|
57
+ status = Faulty::Status.from_entries(
58
+ locked_runs,
59
+ state: state.value,
60
+ lock: lock,
61
+ opened_at: opened_at.value,
62
+ options: circuit_options
63
+ )
64
+ end
65
+
66
+ status
67
+ end
68
+ end
69
+
70
+ # @param options [Hash] Attributes for {Options}
71
+ # @yield [Options] For setting options in a block
72
+ def initialize(**options, &block)
73
+ @circuits = Concurrent::Map.new
74
+ @options = Options.new(options, &block)
75
+ end
76
+
77
+ # Add an entry to storage
78
+ #
79
+ # @see Interface#entry
80
+ # @param (see Interface#entry)
81
+ # @return (see Interface#entry)
82
+ def entry(circuit, time, success)
83
+ memory = fetch(circuit)
84
+ memory.runs.borrow do |runs|
85
+ runs.push([time, success])
86
+ runs.pop if runs.size > options.max_sample_size
87
+ end
88
+ memory.status(circuit.options)
89
+ end
90
+
91
+ # Mark a circuit as open
92
+ #
93
+ # @see Interface#open
94
+ # @param (see Interface#open)
95
+ # @return (see Interface#open)
96
+ def open(circuit, opened_at)
97
+ memory = fetch(circuit)
98
+ opened = memory.state.compare_and_set(:closed, :open)
99
+ memory.opened_at.reset(opened_at) if opened
100
+ opened
101
+ end
102
+
103
+ # Mark a circuit as reopened
104
+ #
105
+ # @see Interface#reopen
106
+ # @param (see Interface#reopen)
107
+ # @return (see Interface#reopen)
108
+ def reopen(circuit, opened_at, previous_opened_at)
109
+ memory = fetch(circuit)
110
+ memory.opened_at.compare_and_set(previous_opened_at, opened_at)
111
+ end
112
+
113
+ # Mark a circuit as closed
114
+ #
115
+ # @see Interface#close
116
+ # @param (see Interface#close)
117
+ # @return (see Interface#close)
118
+ def close(circuit)
119
+ memory = fetch(circuit)
120
+ memory.runs.modify { |_old| [] }
121
+ memory.state.compare_and_set(:open, :closed)
122
+ end
123
+
124
+ # Lock a circuit open or closed
125
+ #
126
+ # @see Interface#lock
127
+ # @param (see Interface#lock)
128
+ # @return (see Interface#lock)
129
+ def lock(circuit, state)
130
+ memory = fetch(circuit)
131
+ memory.lock = state
132
+ end
133
+
134
+ # Unlock a circuit
135
+ #
136
+ # @see Interface#unlock
137
+ # @param (see Interface#unlock)
138
+ # @return (see Interface#unlock)
139
+ def unlock(circuit)
140
+ memory = fetch(circuit)
141
+ memory.lock = nil
142
+ end
143
+
144
+ # Reset a circuit
145
+ #
146
+ # @see Interface#reset
147
+ # @param (see Interface#reset)
148
+ # @return (see Interface#reset)
149
+ def reset(circuit)
150
+ @circuits.delete(circuit.name)
151
+ end
152
+
153
+ # Get the status of a circuit
154
+ #
155
+ # @see Interface#status
156
+ # @param (see Interface#status)
157
+ # @return (see Interface#status)
158
+ def status(circuit)
159
+ fetch(circuit).status(circuit.options)
160
+ end
161
+
162
+ # Get the circuit history up to `max_sample_size`
163
+ #
164
+ # @see Interface#history
165
+ # @param (see Interface#history)
166
+ # @return (see Interface#history)
167
+ def history(circuit)
168
+ fetch(circuit).runs.value
169
+ end
170
+
171
+ # Get a list of circuit names
172
+ #
173
+ # @return [Array<String>] The circuit names
174
+ def list
175
+ @circuits.keys
176
+ end
177
+
178
+ # Memory storage is fault-tolerant by default
179
+ #
180
+ # @return [true]
181
+ def fault_tolerant?
182
+ true
183
+ end
184
+
185
+ private
186
+
187
+ # Fetch circuit storage safely or create it if it doesn't exist
188
+ #
189
+ # @return [MemoryCircuit]
190
+ def fetch(circuit)
191
+ @circuits.compute_if_absent(circuit.name) { MemoryCircuit.new }
192
+ end
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,335 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Faulty
4
+ module Storage
5
+ class Redis # rubocop:disable Metrics/ClassLength
6
+ # Separates the time/status for history entry strings
7
+ ENTRY_SEPARATOR = ':'
8
+
9
+ attr_reader :options
10
+
11
+ # Options for {Redis}
12
+ #
13
+ # @!attribute [r] client
14
+ # @return [Redis,ConnectionPool] The Redis instance or a ConnectionPool
15
+ # used to connect to Redis. Default `::Redis.new`
16
+ # @!attribute [r] key_prefix
17
+ # @return [String] A string prepended to all Redis keys used to store
18
+ # circuit state. Default `faulty`.
19
+ # @!attribute [r] key_separator
20
+ # @return [String] A string used to separate the parts of the Redis keys
21
+ # used to store circuit state. Defaulty `:`.
22
+ # @!attribute [r] max_sample_size
23
+ # @return [Integer] The number of cache run entries to keep in memory
24
+ # for each circuit. Default `100`.
25
+ # @!attribute [r] sample_ttl
26
+ # @return [Integer] The maximum number of seconds to store a
27
+ # circuit run history entry. Default `100`.
28
+ # @!attribute [r] circuit_ttl
29
+ # @return [Integer] The maximum number of seconds to keep a circuit.
30
+ # A value of `nil` disables circuit expiration.
31
+ # Default `604_800` (1 week).
32
+ # @!attribute [r] list_granularity
33
+ # @return [Integer] The number of seconds after which a new set is
34
+ # created to store circuit names. The old set is kept until
35
+ # circuit_ttl expires. Default `3600` (1 hour).
36
+ Options = Struct.new(
37
+ :client,
38
+ :key_prefix,
39
+ :key_separator,
40
+ :max_sample_size,
41
+ :sample_ttl,
42
+ :circuit_ttl,
43
+ :list_granularity
44
+ ) do
45
+ include ImmutableOptions
46
+
47
+ private
48
+
49
+ def defaults
50
+ {
51
+ key_prefix: 'faulty',
52
+ key_separator: ':',
53
+ max_sample_size: 100,
54
+ sample_ttl: 1800,
55
+ circuit_ttl: 604_800,
56
+ list_granularity: 3600
57
+ }
58
+ end
59
+
60
+ def required
61
+ %i[list_granularity]
62
+ end
63
+
64
+ def finalize
65
+ self.client = ::Redis.new unless client
66
+ end
67
+ end
68
+
69
+ # @param options [Hash] Attributes for {Options}
70
+ # @yield [Options] For setting options in a block
71
+ def initialize(**options, &block)
72
+ @options = Options.new(options, &block)
73
+ end
74
+
75
+ # Add an entry to storage
76
+ #
77
+ # @see Interface#entry
78
+ # @param (see Interface#entry)
79
+ # @return (see Interface#entry)
80
+ def entry(circuit, time, success)
81
+ key = entries_key(circuit)
82
+ pipe do |r|
83
+ r.sadd(list_key, circuit.name)
84
+ r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
85
+ r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
86
+ r.ltrim(key, 0, options.max_sample_size - 1)
87
+ r.expire(key, options.sample_ttl) if options.sample_ttl
88
+ end
89
+
90
+ status(circuit)
91
+ end
92
+
93
+ # Mark a circuit as open
94
+ #
95
+ # @see Interface#open
96
+ # @param (see Interface#open)
97
+ # @return (see Interface#open)
98
+ def open(circuit, opened_at)
99
+ redis do |r|
100
+ opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
101
+ r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
102
+ opened
103
+ end
104
+ end
105
+
106
+ # Mark a circuit as reopened
107
+ #
108
+ # @see Interface#reopen
109
+ # @param (see Interface#reopen)
110
+ # @return (see Interface#reopen)
111
+ def reopen(circuit, opened_at, previous_opened_at)
112
+ redis do |r|
113
+ compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
114
+ end
115
+ end
116
+
117
+ # Mark a circuit as closed
118
+ #
119
+ # @see Interface#close
120
+ # @param (see Interface#close)
121
+ # @return (see Interface#close)
122
+ def close(circuit)
123
+ redis do |r|
124
+ closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
125
+ r.del(entries_key(circuit)) if closed
126
+ closed
127
+ end
128
+ end
129
+
130
+ # Lock a circuit open or closed
131
+ #
132
+ # The circuit_ttl does not apply to locks
133
+ #
134
+ # @see Interface#lock
135
+ # @param (see Interface#lock)
136
+ # @return (see Interface#lock)
137
+ def lock(circuit, state)
138
+ redis { |r| r.set(lock_key(circuit), state) }
139
+ end
140
+
141
+ # Unlock a circuit
142
+ #
143
+ # @see Interface#unlock
144
+ # @param (see Interface#unlock)
145
+ # @return (see Interface#unlock)
146
+ def unlock(circuit)
147
+ redis { |r| r.del(lock_key(circuit)) }
148
+ end
149
+
150
+ # Reset a circuit
151
+ #
152
+ # @see Interface#reset
153
+ # @param (see Interface#reset)
154
+ # @return (see Interface#reset)
155
+ def reset(circuit)
156
+ pipe do |r|
157
+ r.del(
158
+ entries_key(circuit),
159
+ opened_at_key(circuit),
160
+ lock_key(circuit)
161
+ )
162
+ r.set(state_key(circuit), 'closed', ex: options.circuit_ttl)
163
+ end
164
+ end
165
+
166
+ # Get the status of a circuit
167
+ #
168
+ # @see Interface#status
169
+ # @param (see Interface#status)
170
+ # @return (see Interface#status)
171
+ def status(circuit)
172
+ futures = {}
173
+ pipe do |r|
174
+ futures[:state] = r.get(state_key(circuit))
175
+ futures[:lock] = r.get(lock_key(circuit))
176
+ futures[:opened_at] = r.get(opened_at_key(circuit))
177
+ futures[:entries] = r.lrange(entries_key(circuit), 0, -1)
178
+ end
179
+
180
+ Faulty::Status.from_entries(
181
+ map_entries(futures[:entries].value),
182
+ state: futures[:state].value&.to_sym || :closed,
183
+ lock: futures[:lock].value&.to_sym,
184
+ opened_at: futures[:opened_at].value ? futures[:opened_at].value.to_i : nil,
185
+ options: circuit.options
186
+ )
187
+ end
188
+
189
+ # Get the circuit history up to `max_sample_size`
190
+ #
191
+ # @see Interface#history
192
+ # @param (see Interface#history)
193
+ # @return (see Interface#history)
194
+ def history(circuit)
195
+ entries = redis { |r| r.lrange(entries_key(circuit), 0, -1) }
196
+ map_entries(entries).reverse
197
+ end
198
+
199
+ def list
200
+ redis { |r| r.sunion(*all_list_keys) }
201
+ end
202
+
203
+ # Redis storage is not fault-tolerant
204
+ #
205
+ # @return [true]
206
+ def fault_tolerant?
207
+ false
208
+ end
209
+
210
+ private
211
+
212
+ # Generate a key from its parts
213
+ #
214
+ # @return [String] The key
215
+ def key(*parts)
216
+ [options.key_prefix, *parts].join(options.key_separator)
217
+ end
218
+
219
+ def ckey(circuit, *parts)
220
+ key('circuit', circuit.name, *parts)
221
+ end
222
+
223
+ # @return [String] The key for circuit state
224
+ def state_key(circuit)
225
+ ckey(circuit, 'state')
226
+ end
227
+
228
+ # @return [String] The key for circuit run history entries
229
+ def entries_key(circuit)
230
+ ckey(circuit, 'entries')
231
+ end
232
+
233
+ # @return [String] The key for circuit locks
234
+ def lock_key(circuit)
235
+ ckey(circuit, 'lock')
236
+ end
237
+
238
+ # @return [String] The key for circuit opened_at
239
+ def opened_at_key(circuit)
240
+ ckey(circuit, 'opened_at')
241
+ end
242
+
243
+ # Get the current key to add circuit names to
244
+ def list_key
245
+ key('list', current_list_block)
246
+ end
247
+
248
+ # Get all active circuit list keys
249
+ #
250
+ # We use a rolling list of redis sets to store circuit names. This way we
251
+ # can maintain this index, while still using Redis to expire old circuits.
252
+ # Whenever we add a circuit to the list, we add it to the current set. A
253
+ # new set is created every `options.list_granularity` seconds.
254
+ #
255
+ # When reading the list, we union all sets together, which gets us the
256
+ # full list.
257
+ #
258
+ # Each set has its own expiration, so that the oldest sets will
259
+ # automatically be deleted from Redis after `options.circuit_ttl`.
260
+ #
261
+ # It is possible for a single circuit name to be a part of many of these
262
+ # sets. This is the space trade-off we make in exchange for automatic
263
+ # expiration.
264
+ #
265
+ # @return [Array<String>] An array of redis keys for circuit name sets
266
+ def all_list_keys
267
+ num_blocks = (options.circuit_ttl.to_f / options.list_granularity).floor + 1
268
+ start_block = current_list_block - num_blocks + 1
269
+ num_blocks.times.map do |i|
270
+ key('list', start_block + i)
271
+ end
272
+ end
273
+
274
+ # Get the block number for the current list set
275
+ #
276
+ # @return [Integer] The current block number
277
+ def current_list_block
278
+ (Faulty.current_time.to_f / options.list_granularity).floor
279
+ end
280
+
281
+ # Set a value in Redis only if it matches a list of current values
282
+ #
283
+ # @param redis [Redis] The redis connection
284
+ # @param key [String] The redis key to CAS
285
+ # @param old [Array<String>] A list of previous values that pass the
286
+ # comparison
287
+ # @param new [String] The new value to set if the compare passes
288
+ # @return [Boolean] True if the value was set to `new`, false if the CAS
289
+ # failed
290
+ def compare_and_set(redis, key, old, new)
291
+ result = redis.watch(key) do
292
+ if old.include?(redis.get(key))
293
+ redis.multi { |m| m.set(key, new) }
294
+ else
295
+ redis.unwatch
296
+ end
297
+ end
298
+
299
+ result[0] == 'OK'
300
+ end
301
+
302
+ # Yield a Redis connection
303
+ #
304
+ # @yield [Redis] Yields the connection to the block
305
+ # @return The value returned from the block
306
+ def redis
307
+ if options.client.respond_to?(:with)
308
+ options.client.with { |redis| yield redis }
309
+ else
310
+ yield options.client
311
+ end
312
+ end
313
+
314
+ # Yield a pipelined Redis connection
315
+ #
316
+ # @yield [Redis::Pipeline] Yields the connection to the block
317
+ # @return [void]
318
+ def pipe
319
+ redis { |r| r.pipelined { |p| yield p } }
320
+ end
321
+
322
+ # Map raw Redis history entries to Faulty format
323
+ #
324
+ # @see Storage::Interface
325
+ # @param raw_entries [Array<String>] The raw Redis entries
326
+ # @return [Array<Array>] The Faulty-formatted entries
327
+ def map_entries(raw_entries)
328
+ raw_entries.map do |e|
329
+ time, state = e.split(ENTRY_SEPARATOR)
330
+ [time.to_i, state == '1']
331
+ end
332
+ end
333
+ end
334
+ end
335
+ end