faulty 0.1.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +49 -0
- data/.rubocop.yml +12 -0
- data/CHANGELOG.md +56 -0
- data/Gemfile +22 -0
- data/README.md +883 -310
- data/bin/check-version +5 -1
- data/bin/console +1 -1
- data/faulty.gemspec +3 -11
- data/lib/faulty.rb +167 -43
- data/lib/faulty/cache.rb +3 -1
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +10 -21
- data/lib/faulty/cache/fault_tolerant_proxy.rb +15 -4
- data/lib/faulty/cache/interface.rb +1 -1
- data/lib/faulty/cache/mock.rb +1 -1
- data/lib/faulty/cache/null.rb +1 -1
- data/lib/faulty/cache/rails.rb +9 -10
- data/lib/faulty/circuit.rb +31 -16
- data/lib/faulty/error.rb +29 -7
- data/lib/faulty/events.rb +1 -1
- data/lib/faulty/events/callback_listener.rb +1 -1
- data/lib/faulty/events/honeybadger_listener.rb +1 -1
- data/lib/faulty/events/listener_interface.rb +1 -1
- data/lib/faulty/events/log_listener.rb +5 -6
- data/lib/faulty/events/notifier.rb +1 -1
- data/lib/faulty/immutable_options.rb +1 -1
- data/lib/faulty/patch.rb +154 -0
- data/lib/faulty/patch/base.rb +46 -0
- data/lib/faulty/patch/redis.rb +60 -0
- data/lib/faulty/result.rb +2 -2
- data/lib/faulty/status.rb +3 -2
- data/lib/faulty/storage.rb +4 -1
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +51 -56
- data/lib/faulty/storage/interface.rb +3 -2
- data/lib/faulty/storage/memory.rb +8 -4
- data/lib/faulty/storage/redis.rb +78 -16
- data/lib/faulty/version.rb +2 -2
- metadata +14 -131
- data/.travis.yml +0 -44
- data/lib/faulty/scope.rb +0 -117
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8ce491fe09e0c6224c3ecc6da2825eeff601f15f7b5bbfbff065f898860fc48
|
4
|
+
data.tar.gz: caf56c1bfc86511d0acb6fbd3f7507d521bcd7b8ac74b47a85ebcbd9a4fdef1b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 89eb97edae88432ea0e6e36a9c0b0f69d4d89075193ffd5a6eb0fe10e24c0983047289ccdb8e363619c4f83b1e80baec9decc34e4fdf029cfbef24189f244138
|
7
|
+
data.tar.gz: 4cabda8638d452bcac0a6375b284b4465d7eb34ed5e4b546a7247c09816eae4480620d69f9777144bd3d3aa8abcc58ca83243037e5d7aac0635e6d01d45ff289
|
@@ -0,0 +1,49 @@
|
|
1
|
+
---
|
2
|
+
name: CI
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
tags: ['v*']
|
6
|
+
branches: [master]
|
7
|
+
pull_request:
|
8
|
+
branches: ['**']
|
9
|
+
jobs:
|
10
|
+
test:
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
strategy:
|
13
|
+
fail-fast: false
|
14
|
+
matrix:
|
15
|
+
redis: [4]
|
16
|
+
ruby: [2.3, 2.4, 2.5, 2.6, 2.7, 3.0, jruby-9.2.10, truffleruby-20.2.0]
|
17
|
+
include:
|
18
|
+
- redis: 3
|
19
|
+
ruby: 2.7
|
20
|
+
services:
|
21
|
+
redis:
|
22
|
+
image: redis
|
23
|
+
ports:
|
24
|
+
- 6379:6379
|
25
|
+
steps:
|
26
|
+
- uses: actions/checkout@v2
|
27
|
+
- uses: ruby/setup-ruby@v1
|
28
|
+
with:
|
29
|
+
ruby-version: ${{ matrix.ruby }}
|
30
|
+
bundler-cache: true
|
31
|
+
- run: bundle exec rubocop
|
32
|
+
if: matrix.ruby == '2.7'
|
33
|
+
- run: bundle exec rspec --format doc
|
34
|
+
- name: Run codacy-coverage-reporter
|
35
|
+
uses: codacy/codacy-coverage-reporter-action@master
|
36
|
+
with:
|
37
|
+
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
38
|
+
coverage-reports: coverage/lcov/faulty.lcov
|
39
|
+
- run: bin/check-version
|
40
|
+
|
41
|
+
release:
|
42
|
+
needs: test
|
43
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
44
|
+
runs-on: ubuntu-latest
|
45
|
+
steps:
|
46
|
+
- uses: actions/checkout@v2
|
47
|
+
- uses: dawidd6/action-publish-gem@v1
|
48
|
+
with:
|
49
|
+
api_key: ${{secrets.RUBYGEMS_API_KEY}}
|
data/.rubocop.yml
CHANGED
@@ -8,6 +8,9 @@ AllCops:
|
|
8
8
|
Layout/ArgumentAlignment:
|
9
9
|
EnforcedStyle: with_fixed_indentation
|
10
10
|
|
11
|
+
Layout/CaseIndentation:
|
12
|
+
EnforcedStyle: end
|
13
|
+
|
11
14
|
Layout/ParameterAlignment:
|
12
15
|
EnforcedStyle: with_fixed_indentation
|
13
16
|
|
@@ -29,6 +32,9 @@ Layout/LineLength:
|
|
29
32
|
Layout/MultilineMethodCallIndentation:
|
30
33
|
EnforcedStyle: indented
|
31
34
|
|
35
|
+
Layout/RescueEnsureAlignment:
|
36
|
+
Enabled: false
|
37
|
+
|
32
38
|
Lint/RaiseException:
|
33
39
|
Enabled: true
|
34
40
|
|
@@ -44,6 +50,9 @@ RSpec/FilePath:
|
|
44
50
|
RSpec/NamedSubject:
|
45
51
|
Enabled: false
|
46
52
|
|
53
|
+
RSpec/MessageSpies:
|
54
|
+
Enabled: false
|
55
|
+
|
47
56
|
RSpec/MultipleExpectations:
|
48
57
|
Enabled: false
|
49
58
|
|
@@ -59,6 +68,9 @@ Metrics/BlockLength:
|
|
59
68
|
Metrics/MethodLength:
|
60
69
|
Max: 30
|
61
70
|
|
71
|
+
Naming/MethodParameterName:
|
72
|
+
MinNameLength: 1
|
73
|
+
|
62
74
|
Style/Documentation:
|
63
75
|
Enabled: false
|
64
76
|
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,59 @@
|
|
1
|
+
## Release v0.5.0
|
2
|
+
|
3
|
+
* Allow creating a new Faulty instance in Faulty#register #24 justinhoward
|
4
|
+
* Add support for patches to core dependencies starting with redis #14 justinhoward
|
5
|
+
* Improve storage #entries performance by returning entries #23 justinhoward
|
6
|
+
|
7
|
+
### Breaking Changes
|
8
|
+
|
9
|
+
* Faulty #[] no longer differentiates between symbols and strings when accessing
|
10
|
+
Faulty instances
|
11
|
+
* Faulty::Storage::Interface must now return a history array instead of a
|
12
|
+
circuit status object. Custom storage backends must be updated.
|
13
|
+
|
14
|
+
## Release v0.4.0
|
15
|
+
|
16
|
+
* Switch from Travis CI to GitHub actions #11 justinhoward
|
17
|
+
* Only run rubocop for Ruby 2.7 in CI #12 justinhoward
|
18
|
+
* Explicitly add support for Redis 3 and 4 #15 justinhoward
|
19
|
+
* Allow setting default circuit options on Faulty instances #16 justinhoward
|
20
|
+
* Switch to codacy for quality metrics #17 justinhoward
|
21
|
+
* Small logic fix to README #19 silasb
|
22
|
+
* Fix Redis storage dependency on ConnectionPool #21 justinhoward
|
23
|
+
* Allow passing custom circuit to AutoWire #22 justinhoward
|
24
|
+
|
25
|
+
### Breaking Changes
|
26
|
+
|
27
|
+
AutoWire.new is replaced with AutoWire.wrap and no longer creates an instance
|
28
|
+
of AutoWire.
|
29
|
+
|
30
|
+
## Release v0.3.0
|
31
|
+
|
32
|
+
* Add tools for backend fault-tolerance #10
|
33
|
+
* CircuitProxy for wrapping storage in an internal circuit
|
34
|
+
* FallbackChain storage backend for falling back to stable storage
|
35
|
+
* Timeout warnings for Redis backend
|
36
|
+
* AutoWire wrappers for automatically configuring storage and cache
|
37
|
+
* Better documentation for fault-tolerance
|
38
|
+
|
39
|
+
## Release v0.2.0
|
40
|
+
|
41
|
+
* Remove Scopes and replace them with Faulty instances #9
|
42
|
+
|
43
|
+
### Breaking Changes
|
44
|
+
|
45
|
+
* `Faulty::Scope` has been removed. Use `Faulty.new` instead.
|
46
|
+
* `Faulty` is now a class, not a module
|
47
|
+
|
48
|
+
## Release v0.1.5
|
49
|
+
|
50
|
+
* Fix redis storage to expire state key when using CAS #8
|
51
|
+
|
52
|
+
## Release v0.1.4
|
53
|
+
|
54
|
+
* Improve spec coverage for supporting classes #6
|
55
|
+
* Fix redis bug where concurrent CAS requests could crash #7
|
56
|
+
|
1
57
|
## Release v0.1.3
|
2
58
|
|
3
59
|
* Fix bug where memory storage would delete the newest entries #5
|
data/Gemfile
CHANGED
@@ -3,3 +3,25 @@
|
|
3
3
|
source 'https://rubygems.org'
|
4
4
|
|
5
5
|
gemspec
|
6
|
+
|
7
|
+
# We add non-essential gems like debugging tools and CI dependencies
|
8
|
+
# here. This also allows us to use conditional dependencies that depend on the
|
9
|
+
# platform
|
10
|
+
|
11
|
+
not_jruby = %i[ruby mingw x64_mingw].freeze
|
12
|
+
|
13
|
+
gem 'activesupport', '>= 4.2'
|
14
|
+
gem 'bundler', '>= 1.17', '< 3'
|
15
|
+
gem 'byebug', platforms: not_jruby
|
16
|
+
gem 'irb', '~> 1.0'
|
17
|
+
gem 'redcarpet', '~> 3.5', platforms: not_jruby
|
18
|
+
gem 'rspec_junit_formatter', '~> 0.4'
|
19
|
+
gem 'simplecov', '>= 0.17.1'
|
20
|
+
# 0.8 is incompatible with simplecov < 0.18
|
21
|
+
# https://github.com/fortissimo1997/simplecov-lcov/pull/25
|
22
|
+
gem 'simplecov-lcov', '~> 0.7', '< 0.8'
|
23
|
+
gem 'yard', '~> 0.9.25', platforms: not_jruby
|
24
|
+
|
25
|
+
if ENV['REDIS_VERSION']
|
26
|
+
gem 'redis', "~> #{ENV['REDIS_VERSION']}"
|
27
|
+
end
|
data/README.md
CHANGED
@@ -1,19 +1,102 @@
|
|
1
1
|
# Faulty
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/faulty.svg)](https://badge.fury.io/rb/faulty)
|
4
|
-
[![
|
5
|
-
[![Code
|
6
|
-
[![
|
4
|
+
[![CI](https://github.com/ParentSquare/faulty/workflows/CI/badge.svg)](https://github.com/ParentSquare/faulty/actions?query=workflow%3ACI+branch%3Amaster)
|
5
|
+
[![Code Quality](https://app.codacy.com/project/badge/Grade/16bb1df1569a4ddba893a866673dac2a)](https://www.codacy.com/gh/ParentSquare/faulty/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ParentSquare/faulty&utm_campaign=Badge_Grade)
|
6
|
+
[![Code Coverage](https://app.codacy.com/project/badge/Coverage/16bb1df1569a4ddba893a866673dac2a)](https://www.codacy.com/gh/ParentSquare/faulty/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ParentSquare/faulty&utm_campaign=Badge_Coverage)
|
7
7
|
[![Inline docs](http://inch-ci.org/github/ParentSquare/faulty.svg?branch=master)](http://inch-ci.org/github/ParentSquare/faulty)
|
8
8
|
|
9
9
|
Fault-tolerance tools for ruby based on [circuit-breakers][martin fowler].
|
10
10
|
|
11
|
+
**Without Faulty**
|
12
|
+
|
13
|
+
External dependencies like APIs can start failing at any time When they do, it
|
14
|
+
could cause cascading failures in your application.
|
15
|
+
|
11
16
|
```ruby
|
12
|
-
|
17
|
+
# The application will always try to execute this even if the API
|
18
|
+
# fails repeatedly
|
19
|
+
api.users
|
20
|
+
```
|
21
|
+
|
22
|
+
**With Faulty**
|
23
|
+
|
24
|
+
Faulty monitors errors inside this block and will "trip" a circuit if your
|
25
|
+
threshold is passed. Once a circuit is tripped, Faulty stops executing this
|
26
|
+
block until it recovers. Your application can detect external failures, and
|
27
|
+
prevent their effects from degrading overall performance.
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
users = Faulty.circuit('api').try_run do
|
31
|
+
|
32
|
+
# If this raises an exception, it counts towards the failure rate
|
33
|
+
# The exceptions that count as failures are configurable
|
34
|
+
# All failures will be sent to your event listeners for monitoring
|
13
35
|
api.users
|
36
|
+
|
14
37
|
end.or_default([])
|
38
|
+
# Here we return a stubbed value so the app can continue to function
|
39
|
+
# Another strategy is just to re-raise the exception so the app can handle it
|
40
|
+
# or use its default error handler
|
15
41
|
```
|
16
42
|
|
43
|
+
See [What is this for?](#what-is-this-for) for a more detailed explanation.
|
44
|
+
Also see "Release It!: Design and Deploy Production-Ready Software" by
|
45
|
+
[Michael T. Nygard][michael nygard] and the
|
46
|
+
[Martin Fowler Article][martin fowler] post on circuit breakers.
|
47
|
+
|
48
|
+
## Contents
|
49
|
+
|
50
|
+
* [Installation](#installation)
|
51
|
+
* [API Docs](#api-docs)
|
52
|
+
* [Setup](#setup)
|
53
|
+
* [Basic Usage](#basic-usage)
|
54
|
+
* [What is this for?](#what-is-this-for-)
|
55
|
+
* [Configuration](#configuration)
|
56
|
+
+ [Configuring the Storage Backend](#configuring-the-storage-backend)
|
57
|
+
- [Memory](#memory)
|
58
|
+
- [Redis](#redis)
|
59
|
+
- [FallbackChain](#fallbackchain)
|
60
|
+
- [Storage::FaultTolerantProxy](#storagefaulttolerantproxy)
|
61
|
+
- [Storage::CircuitProxy](#storagecircuitproxy)
|
62
|
+
+ [Configuring the Cache Backend](#configuring-the-cache-backend)
|
63
|
+
- [Null](#null)
|
64
|
+
- [Rails](#rails)
|
65
|
+
- [Cache::FaultTolerantProxy](#cachefaulttolerantproxy)
|
66
|
+
- [Cache::CircuitProxy](#cachecircuitproxy)
|
67
|
+
+ [Multiple Configurations](#multiple-configurations)
|
68
|
+
- [The default instance](#the-default-instance)
|
69
|
+
- [Multiple Instances](#multiple-instances)
|
70
|
+
- [Standalone Instances](#standalone-instances)
|
71
|
+
* [Working with circuits](#working-with-circuits)
|
72
|
+
+ [Running a Circuit](#running-a-circuit)
|
73
|
+
- [With Exceptions](#with-exceptions)
|
74
|
+
- [With Faulty::Result](#with-faultyresult)
|
75
|
+
+ [Specifying the Captured Errors](#specifying-the-captured-errors)
|
76
|
+
+ [Using the Cache](#using-the-cache)
|
77
|
+
+ [Configuring the Circuit Threshold](#configuring-the-circuit-threshold)
|
78
|
+
- [Rate Threshold](#rate-threshold)
|
79
|
+
- [Sample Threshold](#sample-threshold)
|
80
|
+
- [Cool Down](#cool-down)
|
81
|
+
+ [Circuit Options](#circuit-options)
|
82
|
+
+ [Listing Circuits](#listing-circuits)
|
83
|
+
+ [Locking Circuits](#locking-circuits)
|
84
|
+
* [Patches](#patches)
|
85
|
+
+ [Patch::Redis](#patchredis)
|
86
|
+
* [Event Handling](#event-handling)
|
87
|
+
+ [CallbackListener](#callbacklistener)
|
88
|
+
+ [Other Built-in Listeners](#other-built-in-listeners)
|
89
|
+
+ [Custom Listeners](#custom-listeners)
|
90
|
+
* [How it Works](#how-it-works)
|
91
|
+
+ [Caching](#caching)
|
92
|
+
+ [Fault Tolerance](#fault-tolerance)
|
93
|
+
* [Implementing a Cache Backend](#implementing-a-cache-backend)
|
94
|
+
* [Implementing a Storage Backend](#implementing-a-storage-backend)
|
95
|
+
* [Alternatives](#alternatives)
|
96
|
+
+ [Currently Active](#currently-active)
|
97
|
+
+ [Previous Work](#previous-work)
|
98
|
+
+ [Faulty's Unique Features](#faulty-s-unique-features)
|
99
|
+
|
17
100
|
## Installation
|
18
101
|
|
19
102
|
Add it to your `Gemfile`:
|
@@ -28,8 +111,10 @@ Or install it manually:
|
|
28
111
|
gem install faulty
|
29
112
|
```
|
30
113
|
|
31
|
-
During your app startup, call
|
32
|
-
`
|
114
|
+
During your app startup, call
|
115
|
+
[`Faulty.init`](https://www.rubydoc.info/gems/faulty/Faulty.init).
|
116
|
+
For Rails, you would do this in `config/initializers/faulty.rb`. See
|
117
|
+
[Setup](#setup) for details.
|
33
118
|
|
34
119
|
## API Docs
|
35
120
|
|
@@ -64,66 +149,31 @@ Faulty.init do |config|
|
|
64
149
|
end
|
65
150
|
```
|
66
151
|
|
67
|
-
|
68
|
-
[Global Configuration](#global-configuration) section.
|
69
|
-
|
70
|
-
## What is this for?
|
71
|
-
|
72
|
-
Circuit breakers are a fault-tolerance tool for creating separation between your
|
73
|
-
application and external dependencies. For example, your application may call an
|
74
|
-
external API to send a text message:
|
75
|
-
|
76
|
-
```ruby
|
77
|
-
TextApi.send(message)
|
78
|
-
```
|
79
|
-
|
80
|
-
In normal operation, this API call is very fast. However what if the texting
|
81
|
-
service started hanging? Your application would quickly use up a lot of
|
82
|
-
resources waiting for requests to return from the service. You could consider
|
83
|
-
adding a timeout to your request:
|
84
|
-
|
85
|
-
```ruby
|
86
|
-
TextApi.send(message, timeout: 5)
|
87
|
-
```
|
88
|
-
|
89
|
-
Now your application will terminate requests after 5 seconds, but that could
|
90
|
-
still add up to a lot of resources if you call this thousands of times. Circuit
|
91
|
-
breakers solve this problem.
|
152
|
+
Or use a faulty instance instead for an object-oriented approach
|
92
153
|
|
93
154
|
```ruby
|
94
|
-
Faulty.
|
95
|
-
|
155
|
+
faulty = Faulty.new do
|
156
|
+
config.storage = Faulty::Storage::Redis.new
|
96
157
|
end
|
97
158
|
```
|
98
159
|
|
99
|
-
|
100
|
-
|
101
|
-
(see [How it Works](#how-it-works)), calls to the text API will be paused for
|
102
|
-
the configured cool down period. Your application resources are not overwhelmed.
|
103
|
-
|
104
|
-
You are free to implement a fallback or error handling however you wish, for
|
105
|
-
example, in this case, you might add the text message to a failure queue:
|
106
|
-
|
107
|
-
```ruby
|
108
|
-
Faulty.circuit(:text_api).run do
|
109
|
-
TextApi.send(message, timeout: 5)
|
110
|
-
rescue Faulty::CircuitError => e
|
111
|
-
FailureQueue.enqueue(message)
|
112
|
-
end
|
113
|
-
```
|
160
|
+
For a full list of configuration options, see the
|
161
|
+
[Configuration](#configuration) section.
|
114
162
|
|
115
163
|
## Basic Usage
|
116
164
|
|
117
|
-
To create a circuit, call
|
118
|
-
circuit
|
119
|
-
|
165
|
+
To create a circuit, call
|
166
|
+
[`Faulty.circuit`](https://www.rubydoc.info/gems/faulty/Faulty.circuit).
|
167
|
+
This can be done as you use the circuit, or you can set it up beforehand. Any
|
168
|
+
options passed to the `circuit` method are synchronized across threads and saved
|
169
|
+
as long as the process is alive.
|
120
170
|
|
121
171
|
```ruby
|
122
|
-
circuit1 = Faulty.circuit(:api,
|
172
|
+
circuit1 = Faulty.circuit(:api, rate_threshold: 0.6)
|
123
173
|
|
124
174
|
# The options from above are also used when called here
|
125
175
|
circuit2 = Faulty.circuit(:api)
|
126
|
-
circuit2.options.
|
176
|
+
circuit2.options.rate_threshold == 0.6 # => true
|
127
177
|
|
128
178
|
# The same circuit is returned on each consecutive call
|
129
179
|
circuit1.equal?(circuit2) # => true
|
@@ -140,11 +190,12 @@ end
|
|
140
190
|
See [How it Works](#how-it-works) for more details about how Faulty handles
|
141
191
|
circuit failures.
|
142
192
|
|
143
|
-
If the `run` block above fails, a
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
193
|
+
If the `run` block above fails, a
|
194
|
+
[`Faulty::CircuitError`](https://www.rubydoc.info/gems/faulty/Faulty/CircuitError)
|
195
|
+
will be raised. It is up to your application to handle that error however
|
196
|
+
necessary or crash. Often though, you don't want to crash your application when
|
197
|
+
a circuit fails, but instead apply a fallback or default behavior. For this,
|
198
|
+
Faulty provides the `try_run` method:
|
148
199
|
|
149
200
|
```ruby
|
150
201
|
result = Faulty.circuit(:api).try_run do
|
@@ -158,11 +209,13 @@ else
|
|
158
209
|
end
|
159
210
|
```
|
160
211
|
|
161
|
-
The `try_run`
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
212
|
+
The [`try_run`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:try_run)
|
213
|
+
method returns a result type instead of raising errors. See the API docs for
|
214
|
+
[`Result`](https://www.rubydoc.info/gems/faulty/Faulty/Result) for more
|
215
|
+
information. Here we use it to check whether the result is `ok?` (not an error).
|
216
|
+
If it is we set the users variable, otherwise we set a default of an empty
|
217
|
+
array. This pattern is so common, that `Result` also implements a helper method
|
218
|
+
`or_default` to do the same thing:
|
166
219
|
|
167
220
|
```ruby
|
168
221
|
users = Faulty.circuit(:api).try_run do
|
@@ -170,55 +223,87 @@ users = Faulty.circuit(:api).try_run do
|
|
170
223
|
end.or_default([])
|
171
224
|
```
|
172
225
|
|
173
|
-
|
226
|
+
If you want to globally wrap your core dependencies, like your cache or
|
227
|
+
database, you may want to look at [Patches](#patches), which can automatically
|
228
|
+
wrap your connections in a Faulty circuit.
|
174
229
|
|
175
|
-
|
176
|
-
|
177
|
-
|
230
|
+
See [Running a Circuit](#running-a-circuit) for more in-depth examples. Also,
|
231
|
+
make sure you have proper [Event Handlers](#event-handling) setup so that you
|
232
|
+
can monitor your circuits for failures.
|
178
233
|
|
179
|
-
|
180
|
-
- Integrated caching inspired by Netflix's [Hystrix][hystrix] with automatic
|
181
|
-
cache jitter and error fallback.
|
182
|
-
- Event-based monitoring
|
234
|
+
## What is this for?
|
183
235
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
of runs and the failure rate.
|
236
|
+
Circuit breakers are a fault-tolerance tool for creating separation between your
|
237
|
+
application and external dependencies. For example, your application may call an
|
238
|
+
external API to send a text message:
|
188
239
|
|
189
|
-
|
190
|
-
|
191
|
-
|
240
|
+
```ruby
|
241
|
+
TextApi.send(message)
|
242
|
+
```
|
192
243
|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
244
|
+
In normal operation, this API call is very fast. However what if the texting
|
245
|
+
service started hanging? Your application would quickly use up a lot of
|
246
|
+
resources waiting for requests to return from the service. You could consider
|
247
|
+
adding a timeout to your request:
|
197
248
|
|
198
|
-
|
199
|
-
|
200
|
-
|
249
|
+
```ruby
|
250
|
+
TextApi.send(message, timeout: 5)
|
251
|
+
```
|
201
252
|
|
202
|
-
|
203
|
-
|
204
|
-
|
253
|
+
Now your application will terminate requests after 5 seconds, but that could
|
254
|
+
still add up to a lot of resources if you call this thousands of times. Circuit
|
255
|
+
breakers solve this problem.
|
256
|
+
|
257
|
+
```ruby
|
258
|
+
Faulty.circuit('text_api').run do
|
259
|
+
TextApi.send(message, timeout: 5)
|
260
|
+
end
|
261
|
+
```
|
205
262
|
|
206
|
-
|
263
|
+
Now, when the text API hangs, the first few will run and start timing out. This
|
264
|
+
will trip the circuit. After the circuit trips
|
265
|
+
(see [How it Works](#how-it-works)), calls to the text API will be paused for
|
266
|
+
the configured cool down period. Your application resources are not overwhelmed.
|
267
|
+
|
268
|
+
You are free to implement a fallback or error handling however you wish, for
|
269
|
+
example, in this case, you might add the text message to a failure queue:
|
270
|
+
|
271
|
+
```ruby
|
272
|
+
Faulty.circuit('text_api').run do
|
273
|
+
TextApi.send(message, timeout: 5)
|
274
|
+
rescue Faulty::CircuitError => e
|
275
|
+
FailureQueue.enqueue(message)
|
276
|
+
end
|
277
|
+
```
|
207
278
|
|
208
|
-
|
209
|
-
|
210
|
-
|
279
|
+
## Configuration
|
280
|
+
|
281
|
+
Faulty can be configured with the following configuration options. This example
|
282
|
+
illustrates the default values. In the first example, we configure Faulty
|
283
|
+
globally. The second example shows the same configuration using an instance of
|
284
|
+
Faulty instead of global configuration.
|
211
285
|
|
212
286
|
```ruby
|
213
287
|
Faulty.init do |config|
|
214
288
|
# The cache backend to use. By default, Faulty looks for a Rails cache. If
|
215
289
|
# that's not available, it uses an ActiveSupport::Cache::Memory instance.
|
216
290
|
# Otherwise, it uses a Faulty::Cache::Null and caching is disabled.
|
291
|
+
# Whatever backend is given here is automatically wrapped in
|
292
|
+
# Faulty::Cache::AutoWire. This adds fault-tolerance features, see the
|
293
|
+
# AutoWire API docs for more details.
|
217
294
|
config.cache = Faulty::Cache::Default.new
|
218
295
|
|
296
|
+
# A hash of default options to be used when creating new Circuits.
|
297
|
+
# See Circuit Options below for a full list of these
|
298
|
+
config.circuit_defaults = {}
|
299
|
+
|
219
300
|
# The storage backend. By default, Faulty uses an in-memory store. For most
|
220
301
|
# production applications, you'll want a more robust backend. Faulty also
|
221
302
|
# provides Faulty::Storage::Redis for this.
|
303
|
+
# Whatever backend is given here is automatically wrapped in
|
304
|
+
# Faulty::Storage::AutoWire. This adds fault-tolerance features, see the
|
305
|
+
# AutoWire APi docs for more details. If an array of storage backends is
|
306
|
+
# given, each one will be tried in order until one succeeds.
|
222
307
|
config.storage = Faulty::Storage::Memory.new
|
223
308
|
|
224
309
|
# An array of event listeners. Each object in the array should implement
|
@@ -233,6 +318,25 @@ Faulty.init do |config|
|
|
233
318
|
end
|
234
319
|
```
|
235
320
|
|
321
|
+
Here is the same configuration using an instance of `Faulty`. This is a more
|
322
|
+
object-oriented approach.
|
323
|
+
|
324
|
+
```ruby
|
325
|
+
faulty = Faulty.new do |config|
|
326
|
+
config.cache = Faulty::Cache::Default.new
|
327
|
+
config.storage = Faulty::Storage::Memory.new
|
328
|
+
config.listeners = [Faulty::Events::LogListener.new]
|
329
|
+
config.notifier = Faulty::Events::Notifier.new(config.listeners)
|
330
|
+
end
|
331
|
+
```
|
332
|
+
|
333
|
+
Most of the examples in this README use the global Faulty class methods, but
|
334
|
+
they work the same way when using an instance. Just substitute your instance
|
335
|
+
instead of `Faulty`. There is no preferred way to use Faulty. Choose whichever
|
336
|
+
configuration mechanism works best for your application. Also see
|
337
|
+
[Multiple Configurations](#multiple-configurations) if your application needs
|
338
|
+
to set different options in different scenarios.
|
339
|
+
|
236
340
|
For all Faulty APIs that have configuration, you can also pass in an options
|
237
341
|
hash. For example, `Faulty.init` could be called like this:
|
238
342
|
|
@@ -240,248 +344,546 @@ hash. For example, `Faulty.init` could be called like this:
|
|
240
344
|
Faulty.init(cache: Faulty::Cache::Null.new)
|
241
345
|
```
|
242
346
|
|
243
|
-
|
347
|
+
### Configuring the Storage Backend
|
244
348
|
|
245
|
-
A
|
246
|
-
|
247
|
-
|
248
|
-
within the context of each scope. All options given after the first call to
|
249
|
-
`Faulty.circuit` (or `Scope.circuit`) are ignored.
|
349
|
+
A storage backend is required to use Faulty. By default, it uses in-memory
|
350
|
+
storage, but Redis is also available, along with a number of wrappers used to
|
351
|
+
improve resiliency and fault-tolerance.
|
250
352
|
|
251
|
-
|
252
|
-
read-only once created.
|
353
|
+
#### Memory
|
253
354
|
|
254
|
-
The
|
355
|
+
The
|
356
|
+
[`Faulty::Storage::Memory`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/Memory)
|
357
|
+
backend is the default storage backend. You may prefer this implementation if
|
358
|
+
you want to avoid the complexity and potential failure-mode of cross-network
|
359
|
+
circuit storage. The trade-off is that circuit state is only contained within a
|
360
|
+
single process and will not be saved across application restarts. Locks will
|
361
|
+
also be cleared on restart.
|
255
362
|
|
256
|
-
|
257
|
-
Faulty.circuit(:api) do |config|
|
258
|
-
# The cache backend for this circuit. Inherits the global cache by default.
|
259
|
-
config.cache = Faulty.options.cache
|
363
|
+
The default configuration:
|
260
364
|
|
261
|
-
|
262
|
-
|
263
|
-
config.
|
365
|
+
```ruby
|
366
|
+
Faulty.init do |config|
|
367
|
+
config.storage = Faulty::Storage::Memory.new do |storage|
|
368
|
+
# The maximum number of circuit runs that will be stored
|
369
|
+
storage.max_sample_size = 100
|
370
|
+
end
|
371
|
+
end
|
372
|
+
```
|
264
373
|
|
265
|
-
|
266
|
-
# Caching section for more detail. A value of nil disables cache refreshing.
|
267
|
-
config.cache_refreshes_after = 900
|
374
|
+
#### Redis
|
268
375
|
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
376
|
+
The [`Faulty::Storage::Redis`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/Redis)
|
377
|
+
backend provides distributed circuit storage using Redis. Although Faulty takes
|
378
|
+
steps to reduce risk (See [Fault Tolerance](#fault-tolerance)), using
|
379
|
+
cross-network storage does introduce some additional failure modes. To reduce
|
380
|
+
this risk, be sure to set conservative timeouts for your Redis connection.
|
381
|
+
Setting high timeouts will print warnings to stderr.
|
273
382
|
|
274
|
-
|
275
|
-
# circuit to half-open.
|
276
|
-
config.cool_down = 300
|
383
|
+
The default configuration:
|
277
384
|
|
278
|
-
|
279
|
-
|
280
|
-
config.
|
385
|
+
```ruby
|
386
|
+
Faulty.init do |config|
|
387
|
+
config.storage = Faulty::Storage::Redis.new do |storage|
|
388
|
+
# The Redis client. Accepts either a Redis instance, or a ConnectionPool
|
389
|
+
# of Redis instances. A low timeout is highly recommended to prevent
|
390
|
+
# cascading failures when evaluating circuits.
|
391
|
+
storage.client = ::Redis.new(timeout: 1)
|
281
392
|
|
282
|
-
|
283
|
-
|
393
|
+
# The prefix to prepend to all redis keys used by Faulty circuits
|
394
|
+
storage.key_prefix = 'faulty'
|
284
395
|
|
285
|
-
|
286
|
-
|
396
|
+
# A string to separate the parts of the redis key
|
397
|
+
storage.key_separator = ':'
|
287
398
|
|
288
|
-
|
289
|
-
|
399
|
+
# The maximum number of circuit runs that will be stored
|
400
|
+
storage.max_sample_size = 100
|
290
401
|
|
291
|
-
|
292
|
-
|
402
|
+
# The maximum number of seconds that a circuit run will be stored
|
403
|
+
storage.sample_ttl = 1800
|
293
404
|
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
```
|
405
|
+
# The maximum number of seconds to store a circuit. Does not apply to
|
406
|
+
# locks, which are indefinite.
|
407
|
+
storage.circuit_ttl = 604_800 # 1 Week
|
298
408
|
|
299
|
-
|
300
|
-
|
409
|
+
# The number of seconds between circuit expirations. Changing this setting
|
410
|
+
# is not recommended. See API docs for more implementation details.
|
411
|
+
storage.list_granularity = 3600
|
301
412
|
|
302
|
-
|
303
|
-
|
413
|
+
# If true, disables warnings about recommended client settings like timeouts
|
414
|
+
storage.disable_warnings = false
|
415
|
+
end
|
416
|
+
end
|
304
417
|
```
|
305
418
|
|
306
|
-
|
419
|
+
#### FallbackChain
|
307
420
|
|
308
|
-
Faulty
|
309
|
-
|
310
|
-
|
311
|
-
|
421
|
+
The [`Faulty::Storage::FallbackChain`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/FallbackChain)
|
422
|
+
backend is a wrapper for multiple prioritized storage backends. If the first
|
423
|
+
backend in the chain fails, consecutive backends are tried until one succeeds.
|
424
|
+
The recommended use-case for this is to fall back on reliable storage if a
|
425
|
+
networked storage backend fails.
|
312
426
|
|
313
|
-
|
314
|
-
|
427
|
+
For example, you may configure Redis as your primary storage backend, with an
|
428
|
+
in-memory storage backend as a fallback:
|
315
429
|
|
316
430
|
```ruby
|
317
|
-
|
318
|
-
.
|
319
|
-
|
320
|
-
|
431
|
+
Faulty.init do |config|
|
432
|
+
config.storage = Faulty::Storage::FallbackChain.new([
|
433
|
+
Faulty::Storage::Redis.new,
|
434
|
+
Faulty::Storage::Memory.new
|
435
|
+
])
|
436
|
+
end
|
321
437
|
```
|
322
438
|
|
323
|
-
|
439
|
+
Faulty instances will automatically use a fallback chain if an array is given to
|
440
|
+
the `storage` option, so this example is equivalent to the above:
|
324
441
|
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
`cache_refreshes_after` when determining whether to refresh a cache entry.
|
334
|
-
This mitigates the "thundering herd" effect caused by many processes
|
335
|
-
simultaneously refreshing the cache.
|
442
|
+
```ruby
|
443
|
+
Faulty.init do |config|
|
444
|
+
config.storage = [
|
445
|
+
Faulty::Storage::Redis.new,
|
446
|
+
Faulty::Storage::Memory.new
|
447
|
+
]
|
448
|
+
end
|
449
|
+
```
|
336
450
|
|
337
|
-
|
338
|
-
|
339
|
-
|
451
|
+
If the fallback chain fails-over to backup storage, circuit states will not
|
452
|
+
carry over, so failover could be temporarily disruptive to your application.
|
453
|
+
However, any calls to `#lock` or `#unlock` will always be persisted to all
|
454
|
+
backends so that locks are maintained during failover.
|
340
455
|
|
341
|
-
|
342
|
-
circuit state. If the circuit is closed or half-open, then it will run the
|
343
|
-
block. If the block is successful, then it will update the circuit, write to the
|
344
|
-
cache and return the new value.
|
456
|
+
#### Storage::FaultTolerantProxy
|
345
457
|
|
346
|
-
|
347
|
-
|
458
|
+
This wrapper is applied to all non-fault-tolerant storage backends by default
|
459
|
+
(see the [API docs for `Faulty::Storage::AutoWire`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/AutoWire)).
|
348
460
|
|
349
|
-
|
350
|
-
|
461
|
+
[`Faulty::Storage::FaultTolerantProxy`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/FaultTolerantProxy)
|
462
|
+
is a wrapper that suppresses storage errors and returns sensible defaults during
|
463
|
+
failures. If a storage backend is failing, all circuits will be treated as
|
464
|
+
closed regardless of locks or previous history.
|
351
465
|
|
352
|
-
If
|
353
|
-
|
354
|
-
succeeds, the cache is refreshed. If the block fails, the default of `[]` will
|
355
|
-
be returned.
|
466
|
+
If you wish your application to use a secondary storage backend instead of
|
467
|
+
failing closed, use [`FallbackChain`](#storagefallbackchain).
|
356
468
|
|
357
|
-
|
469
|
+
#### Storage::CircuitProxy
|
358
470
|
|
359
|
-
|
360
|
-
the
|
361
|
-
these errors are sent to the notifier.
|
471
|
+
This wrapper is applied to all non-fault-tolerant storage backends by default
|
472
|
+
(see the [API docs for `Faulty::Storage::AutoWire`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/AutoWire)).
|
362
473
|
|
363
|
-
|
364
|
-
|
474
|
+
[`Faulty::Storage::CircuitProxy`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/CircuitProxy)
|
475
|
+
is a wrapper that uses an independent in-memory circuit to track failures to
|
476
|
+
storage backends. If a storage backend fails continuously, it will be
|
477
|
+
temporarily disabled and raise `Faulty::CircuitError`s.
|
365
478
|
|
366
|
-
|
479
|
+
Typically this is used inside a [`FaultTolerantProxy`](#storagefaulttolerantproxy) or
|
480
|
+
[`FallbackChain`](#storagefallbackchain) so that these storage failures are handled
|
481
|
+
gracefully.
|
367
482
|
|
368
|
-
|
369
|
-
events. The full list of events is available from `Faulty::Events::EVENTS`.
|
483
|
+
### Configuring the Cache Backend
|
370
484
|
|
371
|
-
|
372
|
-
- `circuit_cache_hit` - A circuit hit the cache. Payload: `circuit`, `key`
|
373
|
-
- `circuit_cache_miss` - A circuit hit the cache. Payload: `circuit`, `key`
|
374
|
-
- `circuit_cache_write` - A circuit wrote to the cache. Payload: `circuit`, `key`
|
375
|
-
- `circuit_closed` - A circuit closed. Payload: `circuit`
|
376
|
-
- `circuit_failure` - A circuit execution raised an error. Payload: `circuit`,
|
377
|
-
`status`, `error`
|
378
|
-
- `circuit_opened` - A circuit execution caused the circuit to open. Payload
|
379
|
-
`circuit`, `error`
|
380
|
-
- `circuit_reopened` - A circuit execution cause the circuit to reopen from
|
381
|
-
half-open. Payload: `circuit`, `error`.
|
382
|
-
- `circuit_skipped` - A circuit execution was skipped because the circuit is
|
383
|
-
closed. Payload: `circuit`
|
384
|
-
- `circuit_success` - A circuit execution was successful. Payload: `circuit`,
|
385
|
-
`status`
|
386
|
-
- `storage_failure` - A storage backend raised an error. Payload `circuit` (can
|
387
|
-
be nil), `action`, `error`
|
485
|
+
#### Null
|
388
486
|
|
389
|
-
|
390
|
-
|
487
|
+
The [`Faulty::Cache::Null`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/Null)
|
488
|
+
cache disables caching. It is the default if Rails and ActiveSupport are not
|
489
|
+
present.
|
391
490
|
|
392
|
-
|
491
|
+
#### Rails
|
393
492
|
|
394
|
-
|
395
|
-
|
493
|
+
[`Faulty::Cache::Rails`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/Rails)
|
494
|
+
is the default cache if Rails or ActiveSupport are present. If Rails is present,
|
495
|
+
it uses `Rails.cache` as the backend. If ActiveSupport is present, but Rails is
|
496
|
+
not, it creates a new `ActiveSupport::Cache::MemoryStore` by default. This
|
497
|
+
backend can be used with any `ActiveSupport::Cache`.
|
396
498
|
|
397
499
|
```ruby
|
398
500
|
Faulty.init do |config|
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
MyNotifier.alert("Circuit #{payload[:circuit].name} opened: #{payload[:error].message}")
|
403
|
-
end
|
404
|
-
end
|
405
|
-
config.listeners = [listener]
|
501
|
+
config.cache = Faulty::Cache::Rails.new(
|
502
|
+
ActiveSupport::Cache::RedisCacheStore.new
|
503
|
+
)
|
406
504
|
end
|
407
505
|
```
|
408
506
|
|
409
|
-
|
507
|
+
#### Cache::FaultTolerantProxy
|
410
508
|
|
411
|
-
|
412
|
-
|
413
|
-
and reporting software.
|
509
|
+
This wrapper is applied to all non-fault-tolerant cache backends by default
|
510
|
+
(see the API docs for `Faulty::Cache::AutoWire`).
|
414
511
|
|
415
|
-
|
416
|
-
|
512
|
+
[`Faulty::Cache::FaultTolerantProxy`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/FaultTolerantProxy)
|
513
|
+
is a wrapper that suppresses cache errors and acts like a null cache during
|
514
|
+
failures. Reads always return `nil`, and writes are no-ops.
|
417
515
|
|
418
|
-
|
516
|
+
#### Cache::CircuitProxy
|
419
517
|
|
420
|
-
|
421
|
-
`Faulty::
|
518
|
+
This wrapper is applied to all non-fault-tolerant circuit backends by default
|
519
|
+
(see the API docs for `Faulty::Circuit::AutoWire`).
|
520
|
+
|
521
|
+
[`Faulty::Cache::CircuitProxy`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/CircuitProxy)
|
522
|
+
is a wrapper that uses an independent in-memory circuit to track failures to
|
523
|
+
cache backends. If a cache backend fails continuously, it will be
|
524
|
+
temporarily disabled and raise `Faulty::CircuitError`s.
|
525
|
+
|
526
|
+
Typically this is used inside a
|
527
|
+
[`FaultTolerantProxy`](#cachefaulttolerantproxy) so that these cache failures
|
528
|
+
are handled gracefully.
|
529
|
+
|
530
|
+
### Multiple Configurations
|
531
|
+
|
532
|
+
It is possible to have multiple configurations of Faulty running within the same
|
533
|
+
process. The most common setup is to simply use `Faulty.init` to
|
534
|
+
configure Faulty globally, however it is possible to have additional
|
535
|
+
configurations.
|
536
|
+
|
537
|
+
#### The default instance
|
538
|
+
|
539
|
+
When you call [`Faulty.init`](https://www.rubydoc.info/gems/faulty/Faulty.init),
|
540
|
+
you are actually creating the default instance of `Faulty`. You can access this
|
541
|
+
instance directly by calling
|
542
|
+
[`Faulty.default`](https://www.rubydoc.info/gems/faulty/Faulty.default).
|
422
543
|
|
423
544
|
```ruby
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
545
|
+
# We create the default instance
|
546
|
+
Faulty.init
|
547
|
+
|
548
|
+
# Access the default instance
|
549
|
+
faulty = Faulty.default
|
550
|
+
|
551
|
+
# Alternatively, access the instance by name
|
552
|
+
faulty = Faulty[:default]
|
553
|
+
```
|
554
|
+
|
555
|
+
You can rename the default instance if desired:
|
556
|
+
|
557
|
+
```ruby
|
558
|
+
Faulty.init(:custom_default)
|
559
|
+
|
560
|
+
instance = Faulty.default
|
561
|
+
instance = Faulty[:custom_default]
|
562
|
+
```
|
563
|
+
|
564
|
+
#### Multiple Instances
|
565
|
+
|
566
|
+
If you want multiple instance, but want global, thread-safe access to
|
567
|
+
them, you can use
|
568
|
+
[`Faulty.register`](https://www.rubydoc.info/gems/faulty/Faulty.register):
|
569
|
+
|
570
|
+
```ruby
|
571
|
+
api_faulty = Faulty.new do |config|
|
572
|
+
# This accepts the same options as Faulty.init
|
428
573
|
end
|
574
|
+
|
575
|
+
Faulty.register(:api, api_faulty)
|
576
|
+
|
577
|
+
# Now access the instance globally
|
578
|
+
Faulty[:api]
|
429
579
|
```
|
430
580
|
|
581
|
+
When you call [`Faulty.circuit`](https://www.rubydoc.info/gems/faulty/Faulty.circuit),
|
582
|
+
that's the same as calling `Faulty.default.circuit`, so you can apply the same
|
583
|
+
principal to any other registered Faulty instance:
|
584
|
+
|
431
585
|
```ruby
|
432
|
-
Faulty.
|
433
|
-
|
586
|
+
Faulty[:api].circuit('api_circuit').run { 'ok' }
|
587
|
+
```
|
588
|
+
|
589
|
+
You can also create and register a Faulty instance in one step:
|
590
|
+
|
591
|
+
```ruby
|
592
|
+
Faulty.register(:api) do |config|
|
593
|
+
# This accepts the same options as Faulty.init
|
434
594
|
end
|
435
595
|
```
|
436
596
|
|
437
|
-
|
597
|
+
#### Standalone Instances
|
438
598
|
|
439
|
-
|
599
|
+
If you choose, you can use Faulty instances without registering them globally by
|
600
|
+
simply calling [`Faulty.new`](https://www.rubydoc.info/gems/faulty/Faulty:initialize).
|
601
|
+
This is more object-oriented and is necessary if you use dependency injection.
|
602
|
+
|
603
|
+
```ruby
|
604
|
+
faulty = Faulty.new
|
605
|
+
faulty.circuit('standalone_circuit')
|
606
|
+
```
|
607
|
+
|
608
|
+
Calling `#circuit` on the instance still has the same memoization behavior that
|
609
|
+
`Faulty.circuit` has, so subsequent calls to the same circuit will return a
|
610
|
+
memoized circuit object.
|
440
611
|
|
441
|
-
|
442
|
-
|
612
|
+
|
613
|
+
## Working with circuits
|
614
|
+
|
615
|
+
A circuit can be created by calling the `#circuit` method on `Faulty`, or on
|
616
|
+
your Faulty instance:
|
443
617
|
|
444
618
|
```ruby
|
445
|
-
Faulty
|
446
|
-
|
447
|
-
|
448
|
-
|
619
|
+
# With global Faulty configuration
|
620
|
+
circuit = Faulty.circuit('api')
|
621
|
+
|
622
|
+
# Or with a Faulty instance
|
623
|
+
circuit = faulty.circuit('api')
|
624
|
+
```
|
625
|
+
|
626
|
+
### Running a Circuit
|
627
|
+
|
628
|
+
You can handle circuit errors either with exceptions, or with a Faulty
|
629
|
+
[`Result`](https://www.rubydoc.info/gems/faulty/Faulty/Result). They both have
|
630
|
+
the same behavior, but you can choose whatever syntax is more convenient for
|
631
|
+
your use-case.
|
632
|
+
|
633
|
+
#### With Exceptions
|
634
|
+
|
635
|
+
If we want exceptions to be raised, we use the
|
636
|
+
[`#run`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:run) method. This
|
637
|
+
does not suppress exceptions, only monitors them. If `api.users` raises an
|
638
|
+
exception here, it will bubble up to the caller. The exception will be a
|
639
|
+
sub-class of [`Faulty::CircuitError`](https://www.rubydoc.info/gems/faulty/Faulty/CircuitError),
|
640
|
+
and the error `cause` will be the original error object.
|
641
|
+
|
642
|
+
```ruby
|
643
|
+
begin
|
644
|
+
Faulty.circuit('api').run do
|
645
|
+
api.users
|
449
646
|
end
|
647
|
+
rescue Faulty::CircuitError => e
|
648
|
+
e.cause # The original error
|
450
649
|
end
|
451
650
|
```
|
452
651
|
|
453
|
-
|
652
|
+
#### With Faulty::Result
|
454
653
|
|
455
|
-
|
456
|
-
|
654
|
+
Sometimes exception handling is awkward to deal with, and could cause a lot of
|
655
|
+
extra boilerplate code. In simple cases, it's can be more concise to allow
|
656
|
+
Faulty to capture exceptions. Use the
|
657
|
+
[`#try_run`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:try_run) method
|
658
|
+
for this.
|
457
659
|
|
458
660
|
```ruby
|
459
|
-
Faulty.
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
storage.client = ::Redis.new
|
661
|
+
result = Faulty.circuit('api').try_run do
|
662
|
+
api.users
|
663
|
+
end
|
664
|
+
```
|
464
665
|
|
465
|
-
|
466
|
-
|
666
|
+
The `result` variable is an instance of
|
667
|
+
[`Faulty::Result`](https://www.rubydoc.info/gems/faulty/Faulty/Result). A result
|
668
|
+
can either be an error if the circuit failed, or an "ok" value if it succeeded.
|
467
669
|
|
468
|
-
|
469
|
-
storage.key_separator = ':'
|
670
|
+
You can check whether it's an error with the `ok?` or `error?` method.
|
470
671
|
|
471
|
-
|
472
|
-
|
672
|
+
```ruby
|
673
|
+
if result.ok?
|
674
|
+
users = result.get
|
675
|
+
else
|
676
|
+
error = result.error
|
677
|
+
end
|
678
|
+
```
|
473
679
|
|
474
|
-
|
475
|
-
|
476
|
-
|
680
|
+
Sometimes you want your application to crash when a circuit fails, but other
|
681
|
+
times, you might want to return a default or fallback value. The `Result` object
|
682
|
+
has a method [`#or_default`](https://www.rubydoc.info/gems/faulty/Faulty/Result:or_default)
|
683
|
+
to do that.
|
684
|
+
|
685
|
+
```ruby
|
686
|
+
# Users will be nil if the result is an error
|
687
|
+
users = result.or_default
|
688
|
+
|
689
|
+
# Users will be an empty array if the result is an error
|
690
|
+
users = result.or_default([])
|
691
|
+
|
692
|
+
# Users will be the return value of the block
|
693
|
+
users = result.or_default do
|
694
|
+
# ...
|
695
|
+
end
|
696
|
+
```
|
697
|
+
|
698
|
+
As we showed in the [Basic Usage](#basic-usage) section, you can put this
|
699
|
+
together in a nice one-liner.
|
700
|
+
|
701
|
+
```ruby
|
702
|
+
Faulty.circuit('api').try_run { api.users }.or_default([])
|
703
|
+
```
|
704
|
+
|
705
|
+
### Specifying the Captured Errors
|
706
|
+
|
707
|
+
By default, Faulty circuits will capture all `StandardError` errors, but
|
708
|
+
sometimes you might not want every error to count as a circuit failure. For
|
709
|
+
example, an HTTP 404 Not Found response typically should not cause a circuit to
|
710
|
+
fail. You can customize the errors that Faulty captures
|
711
|
+
|
712
|
+
```ruby
|
713
|
+
Faulty.circuit('api', errors: [Net::HTTPServerException]).run do
|
714
|
+
# If this raises any exception other than Net::HTTPServerException
|
715
|
+
# Faulty will not capture it at all, and it will not count as a circuit failure
|
716
|
+
api.find_user(3)
|
717
|
+
end
|
718
|
+
```
|
719
|
+
|
720
|
+
Or, if you'd instead like to specify errors to be excluded:
|
721
|
+
|
722
|
+
```ruby
|
723
|
+
Faulty.circuit('api', exclude: [Net::HTTPClientException]).run do
|
724
|
+
# If this raises a Net::HTTPClientException, Faulty will not capture it
|
725
|
+
api.find_user(3)
|
726
|
+
end
|
727
|
+
```
|
728
|
+
|
729
|
+
Both options can even be specified together.
|
730
|
+
|
731
|
+
```ruby
|
732
|
+
Faulty.circuit(
|
733
|
+
'api',
|
734
|
+
errors: [ActiveRecord::ActiveRecordError]
|
735
|
+
exclude: [ActiveRecord::RecordNotFound, ActiveRecord::RecordNotUnique]
|
736
|
+
).run do
|
737
|
+
# This only captures ActiveRecord::ActiveRecordError errors, but not
|
738
|
+
# ActiveRecord::RecordNotFound or ActiveRecord::RecordNotUnique errors
|
739
|
+
user = User.find(3)
|
740
|
+
user.save!
|
741
|
+
end
|
742
|
+
```
|
743
|
+
|
744
|
+
### Using the Cache
|
745
|
+
|
746
|
+
Circuit runs can be given a cache key, and if they are, the result of the circuit
|
747
|
+
block will be cached. Calls to that circuit block will try to fetch from the
|
748
|
+
cache, and only execute the block if the cache misses.
|
749
|
+
|
750
|
+
```ruby
|
751
|
+
Faulty.circuit('api').run(cache: 'all_users') do
|
752
|
+
api.users
|
753
|
+
end
|
754
|
+
```
|
755
|
+
|
756
|
+
The cache will be refreshed (meaning the circuit will be allowed to execute)
|
757
|
+
after `cache_refreshes_after` (default 900 second). However, the value remains
|
758
|
+
stored in the cache for `cache_expires_in` (default 86400 seconds, 1 day). If
|
759
|
+
the circuit fails, the last cached value will be returned even if
|
760
|
+
`cache_refreshes_after` has passed.
|
761
|
+
|
762
|
+
See the [Caching](#caching) section for more details on Faulty's caching
|
763
|
+
strategy.
|
764
|
+
|
765
|
+
### Configuring the Circuit Threshold
|
766
|
+
|
767
|
+
To configure how a circuit responds to error, use the `cool_down`,
|
768
|
+
`rate_threshold` and `sample_threshold` options.
|
769
|
+
|
770
|
+
#### Rate Threshold
|
771
|
+
|
772
|
+
The first option to look at is `rate_threshold`. This specifies the percentage
|
773
|
+
of circuit runs that must fail before a circuit is opened.
|
774
|
+
|
775
|
+
```ruby
|
776
|
+
# This circuit must fail 70% of the time before the circuit will be tripped
|
777
|
+
Faulty.circuit('api', rate_threshold: 0.7).run { api.users }
|
778
|
+
```
|
779
|
+
|
780
|
+
#### Sample Threshold
|
781
|
+
|
782
|
+
We typically don't want circuits to trip immediately if the first execution
|
783
|
+
fails. This is why we have the `sample_threshold` option. The circuit will never
|
784
|
+
be tripped until we record at least this number of executions.
|
785
|
+
|
786
|
+
```ruby
|
787
|
+
# This circuit must run 10 times before it is allowed to trip. Those 10 runs
|
788
|
+
# can be successes or fails. If at least 70% of them are failures, the circuit
|
789
|
+
# will be opened.
|
790
|
+
Faulty.circuit('api', sample_threshold: 10, rate_threshold: 0.7).run { api.users }
|
791
|
+
```
|
792
|
+
|
793
|
+
#### Cool Down
|
794
|
+
|
795
|
+
The `cool_down` option specifies how much time to wait after a circuit is
|
796
|
+
opened. During this period, the circuit will not be executed. After the cool
|
797
|
+
down elapses, the circuit enters the "half open" state, and execution can be
|
798
|
+
retried. See [How it Works](#how-it-works).
|
799
|
+
|
800
|
+
```ruby
|
801
|
+
# If this circuit trips, it will skip executions for 120 seconds before retrying
|
802
|
+
Faulty.circuit('api', cool_down: 120).run { api.users }
|
803
|
+
```
|
804
|
+
|
805
|
+
### Circuit Options
|
806
|
+
|
807
|
+
A circuit can be created with the following configuration options. Those options
|
808
|
+
are only set once, synchronized across threads, and will persist in-memory until
|
809
|
+
the process exits. If you're using [multiple configurations](#multiple-configurations),
|
810
|
+
the options are retained within the context of each instance. All options given
|
811
|
+
after the first call to `Faulty.circuit` (or `Faulty#circuit`) are ignored.
|
812
|
+
|
813
|
+
```ruby
|
814
|
+
Faulty.circuit('api', rate_threshold: 0.7)
|
815
|
+
|
816
|
+
# These options are ignored since with already initialized the circuit
|
817
|
+
circuit = Faulty.circuit('api', rate_threshold: 0.3)
|
818
|
+
circuit.options.rate_threshold # => 0.7
|
819
|
+
```
|
820
|
+
|
821
|
+
This is because the circuit objects themselves are internally memoized, and are
|
822
|
+
read-only once created.
|
823
|
+
|
824
|
+
The following example represents the defaults for a new circuit:
|
825
|
+
|
826
|
+
```ruby
|
827
|
+
Faulty.circuit('api') do |config|
|
828
|
+
# The cache backend for this circuit. Inherits the global cache by default.
|
829
|
+
config.cache = Faulty.options.cache
|
830
|
+
|
831
|
+
# The number of seconds before a cache entry is expired. After this time, the
|
832
|
+
# cache entry may be fully deleted. If set to nil, the cache will not expire.
|
833
|
+
config.cache_expires_in = 86400
|
834
|
+
|
835
|
+
# The number of seconds before a cache entry should be refreshed. See the
|
836
|
+
# Caching section for more detail. A value of nil disables cache refreshing.
|
837
|
+
config.cache_refreshes_after = 900
|
838
|
+
|
839
|
+
# The number of seconds to add or subtract from cache_refreshes_after
|
840
|
+
# when determining whether a cache entry should be refreshed. Helps mitigate
|
841
|
+
# the "thundering herd" effect
|
842
|
+
config.cache_refresh_jitter = 0.2 * config.cache_refreshes_after
|
843
|
+
|
844
|
+
# After a circuit is opened, the number of seconds to wait before moving the
|
845
|
+
# circuit to half-open.
|
846
|
+
config.cool_down = 300
|
847
|
+
|
848
|
+
# The number of seconds of history that is considered when calculating
|
849
|
+
# the circuit failure rate. The length of the sliding window.
|
850
|
+
config.evaluation_window = 60
|
851
|
+
|
852
|
+
# The errors that will be captured by Faulty and used to trigger circuit
|
853
|
+
# state changes.
|
854
|
+
config.errors = [StandardError]
|
855
|
+
|
856
|
+
# Errors that should be ignored by Faulty and not captured.
|
857
|
+
config.exclude = []
|
858
|
+
|
859
|
+
# The event notifier. Inherits the Faulty instance notifier by default
|
860
|
+
config.notifier = Faulty.options.notifier
|
861
|
+
|
862
|
+
# The minimum failure rate required to trip a circuit
|
863
|
+
config.rate_threshold = 0.5
|
864
|
+
|
865
|
+
# The minimum number of runs required before a circuit can trip
|
866
|
+
config.sample_threshold = 3
|
867
|
+
|
868
|
+
# The storage backend for this circuit. Inherits the Faulty instance storage
|
869
|
+
# by default
|
870
|
+
config.storage = Faulty.options.storage
|
477
871
|
end
|
478
872
|
```
|
479
873
|
|
480
|
-
|
874
|
+
Following the same convention as `Faulty.init`, circuits can also be created
|
875
|
+
with an options hash:
|
876
|
+
|
877
|
+
```ruby
|
878
|
+
Faulty.circuit(:api, cache_expires_in: 1800)
|
879
|
+
```
|
880
|
+
|
881
|
+
### Listing Circuits
|
481
882
|
|
482
883
|
For monitoring or debugging, you may need to retrieve a list of all circuit
|
483
|
-
names. This is possible with `Faulty.list_circuits`
|
484
|
-
|
884
|
+
names. This is possible with [`Faulty.list_circuits`](https://www.rubydoc.info/gems/faulty/Faulty.list_circuits)
|
885
|
+
(or [`Faulty#list_circuits`](https://www.rubydoc.info/gems/faulty/Faulty:list_circuits)
|
886
|
+
if you're using an instance).
|
485
887
|
|
486
888
|
You can get a list of all circuit statuses by mapping those names to their
|
487
889
|
status objects. Be careful though, since this could cause performance issues for
|
@@ -493,7 +895,7 @@ statuses = Faulty.list_circuits.map do |name|
|
|
493
895
|
end
|
494
896
|
```
|
495
897
|
|
496
|
-
|
898
|
+
### Locking Circuits
|
497
899
|
|
498
900
|
It is possible to lock a circuit open or closed. A circuit that is locked open
|
499
901
|
will never execute its block, and always raise an `Faulty::OpenCircuitError`.
|
@@ -501,8 +903,12 @@ This is useful in cases where you need to manually disable a dependency
|
|
501
903
|
entirely. If a cached value is available, that will be returned from the circuit
|
502
904
|
until it expires, even outside its refresh period.
|
503
905
|
|
906
|
+
* [`lock_open!`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:lock_open!)
|
907
|
+
* [`lock_closed!`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:lock_closed!)
|
908
|
+
* [`unlock!`](https://www.rubydoc.info/gems/faulty/Faulty/Circuit:unlock!)
|
909
|
+
|
504
910
|
```ruby
|
505
|
-
Faulty.circuit(
|
911
|
+
Faulty.circuit('broken_api').lock_open!
|
506
912
|
```
|
507
913
|
|
508
914
|
A circuit that is locked closed will never trip. This is useful in cases where a
|
@@ -510,94 +916,258 @@ circuit is continuously tripping incorrectly. If a cached value is available, it
|
|
510
916
|
will have the same behavior as an unlocked circuit.
|
511
917
|
|
512
918
|
```ruby
|
513
|
-
Faulty.circuit(
|
919
|
+
Faulty.circuit('false_positive').lock_closed!
|
514
920
|
```
|
515
921
|
|
516
922
|
To remove a lock of either type:
|
517
923
|
|
518
924
|
```ruby
|
519
|
-
Faulty.circuit(
|
925
|
+
Faulty.circuit('fixed').unlock!
|
520
926
|
```
|
521
927
|
|
522
928
|
Locking or unlocking a circuit has no concurrency guarantees, so it's not
|
523
929
|
recommended to lock or unlock circuits from production code. Instead, locks are
|
524
930
|
intended as an emergency tool for troubleshooting and debugging.
|
525
931
|
|
526
|
-
##
|
932
|
+
## Patches
|
527
933
|
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
934
|
+
For certain core dependencies like a cache or a database connection, it is
|
935
|
+
inconvenient to wrap every call in its own circuit. Faulty provides some patches
|
936
|
+
to wrap these calls in a circuit automatically. To use a patch, it first needs
|
937
|
+
to be loaded. Since patches modify third-party code, they are not automatically
|
938
|
+
required with the Faulty gem, so they need to be required individually.
|
532
939
|
|
533
|
-
|
940
|
+
```ruby
|
941
|
+
require 'faulty'
|
942
|
+
require 'faulty/patch/redis'
|
943
|
+
```
|
534
944
|
|
535
|
-
|
536
|
-
can access this scope directly by calling `Faulty.default`.
|
945
|
+
Or require them in your `Gemfile`
|
537
946
|
|
538
947
|
```ruby
|
539
|
-
|
540
|
-
|
948
|
+
gem 'faulty', require: %w[faulty faulty/patch/redis]
|
949
|
+
```
|
541
950
|
|
542
|
-
|
543
|
-
scope = Faulty.default
|
951
|
+
### Patch::Redis
|
544
952
|
|
545
|
-
|
546
|
-
|
547
|
-
|
953
|
+
[`Faulty::Patch::Redis`](https://www.rubydoc.info/gems/faulty/Faulty/Patch/Redis)
|
954
|
+
protects a Redis client with an internal circuit. Pass a `:faulty` key along
|
955
|
+
with your connection options to enable the circuit breaker.
|
548
956
|
|
549
|
-
|
957
|
+
Keep in mind that when using this patch, you'll most likely want to use the
|
958
|
+
in-memory circuit storage adapter and not the Redis storage adapter. That way
|
959
|
+
if Redis fails, your circuit storage doesn't also fail.
|
550
960
|
|
551
961
|
```ruby
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
962
|
+
require 'faulty/patch/redis'
|
963
|
+
|
964
|
+
redis = Redis.new(url: 'redis://localhost:6379', faulty: {
|
965
|
+
# The name for the redis circuit
|
966
|
+
name: 'redis'
|
967
|
+
|
968
|
+
# The faulty instance to use
|
969
|
+
# This can also be a registered faulty instance or a constant name. See API
|
970
|
+
# docs for more details
|
971
|
+
instance: Faulty.default
|
972
|
+
|
973
|
+
# By default, circuit errors will be subclasses of Redis::BaseError
|
974
|
+
# To disable this behavior, set patch_errors to false and Faulty
|
975
|
+
# will raise its default errors
|
976
|
+
patch_errors: true
|
977
|
+
})
|
978
|
+
redis.connect # raises Faulty::CircuitError if connection fails
|
979
|
+
|
980
|
+
# If the faulty key is not given, no circuit is used
|
981
|
+
redis = Redis.new(url: 'redis://localhost:6379')
|
982
|
+
redis.connect # not protected by a circuit
|
556
983
|
```
|
557
984
|
|
558
|
-
|
985
|
+
## Event Handling
|
986
|
+
|
987
|
+
Faulty uses an event-dispatching model to deliver notifications of internal
|
988
|
+
events. The full list of events is available from
|
989
|
+
[`Faulty::Events::EVENTS`](https://www.rubydoc.info/gems/faulty/Faulty/Events).
|
990
|
+
|
991
|
+
- `cache_failure` - A cache backend raised an error. Payload: `key`, `action`, `error`
|
992
|
+
- `circuit_cache_hit` - A circuit hit the cache. Payload: `circuit`, `key`
|
993
|
+
- `circuit_cache_miss` - A circuit hit the cache. Payload: `circuit`, `key`
|
994
|
+
- `circuit_cache_write` - A circuit wrote to the cache. Payload: `circuit`, `key`
|
995
|
+
- `circuit_closed` - A circuit closed. Payload: `circuit`
|
996
|
+
- `circuit_failure` - A circuit execution raised an error. Payload: `circuit`,
|
997
|
+
`status`, `error`
|
998
|
+
- `circuit_opened` - A circuit execution caused the circuit to open. Payload
|
999
|
+
`circuit`, `error`
|
1000
|
+
- `circuit_reopened` - A circuit execution cause the circuit to reopen from
|
1001
|
+
half-open. Payload: `circuit`, `error`.
|
1002
|
+
- `circuit_skipped` - A circuit execution was skipped because the circuit is
|
1003
|
+
closed. Payload: `circuit`
|
1004
|
+
- `circuit_success` - A circuit execution was successful. Payload: `circuit`,
|
1005
|
+
`status`
|
1006
|
+
- `storage_failure` - A storage backend raised an error. Payload `circuit` (can
|
1007
|
+
be nil), `action`, `error`
|
1008
|
+
|
1009
|
+
By default events are logged using `Faulty::Events::LogListener`, but that can
|
1010
|
+
be replaced, or additional listeners can be added.
|
1011
|
+
|
1012
|
+
### CallbackListener
|
559
1013
|
|
560
|
-
|
561
|
-
|
1014
|
+
The [`CallbackListener`](https://www.rubydoc.info/gems/faulty/Faulty/Events/CallbackListener)
|
1015
|
+
is useful for ad-hoc handling of events. You can specify an event handler by
|
1016
|
+
calling a method on the callback handler by the same name.
|
562
1017
|
|
563
1018
|
```ruby
|
564
|
-
|
565
|
-
#
|
1019
|
+
Faulty.init do |config|
|
1020
|
+
# Replace the default listener with a custom callback listener
|
1021
|
+
listener = Faulty::Events::CallbackListener.new do |events|
|
1022
|
+
events.circuit_opened do |payload|
|
1023
|
+
MyNotifier.alert("Circuit #{payload[:circuit].name} opened: #{payload[:error].message}")
|
1024
|
+
end
|
1025
|
+
end
|
1026
|
+
config.listeners = [listener]
|
566
1027
|
end
|
1028
|
+
```
|
567
1029
|
|
568
|
-
|
1030
|
+
### Other Built-in Listeners
|
569
1031
|
|
570
|
-
|
571
|
-
|
572
|
-
|
1032
|
+
In addition to the log and callback listeners, Faulty intends to implement
|
1033
|
+
built-in service-specific handlers to make it easy to integrate with monitoring
|
1034
|
+
and reporting software.
|
573
1035
|
|
574
|
-
|
575
|
-
|
576
|
-
|
1036
|
+
* [`Faulty::Events::LogListener`](https://www.rubydoc.info/gems/faulty/Faulty/Events/LogListener):
|
1037
|
+
Logs all circuit events to a specified `Logger` or `$stderr` by default. This
|
1038
|
+
is enabled by default if no listeners are specified.
|
1039
|
+
* [`Faulty::Events::HoneybadgerListener`](https://www.rubydoc.info/gems/faulty/Faulty/Events/HoneybadgerListener):
|
1040
|
+
Reports circuit and backend errors to the Honeybadger error reporting service.
|
1041
|
+
|
1042
|
+
If your favorite monitoring software is not supported here, please open a PR
|
1043
|
+
that implements a listener for it.
|
1044
|
+
|
1045
|
+
### Custom Listeners
|
1046
|
+
|
1047
|
+
You can implement your own listener by following the documentation in
|
1048
|
+
[`Faulty::Events::ListenerInterface`](https://www.rubydoc.info/gems/faulty/Faulty/Events/ListenerInterface).
|
1049
|
+
For example:
|
577
1050
|
|
578
1051
|
```ruby
|
579
|
-
|
1052
|
+
class MyFaultyListener
|
1053
|
+
def handle(event, payload)
|
1054
|
+
MyNotifier.alert(event, payload)
|
1055
|
+
end
|
1056
|
+
end
|
580
1057
|
```
|
581
1058
|
|
582
|
-
|
1059
|
+
```ruby
|
1060
|
+
Faulty.init do |config|
|
1061
|
+
config.listeners = [MyFaultyListener.new]
|
1062
|
+
end
|
1063
|
+
```
|
1064
|
+
|
1065
|
+
## How it Works
|
1066
|
+
|
1067
|
+
Faulty implements a version of circuit breakers inspired by "Release It!: Design
|
1068
|
+
and Deploy Production-Ready Software" by [Michael T. Nygard][michael nygard] and
|
1069
|
+
[Martin Fowler's post][martin fowler] on the subject. A few notable features of
|
1070
|
+
Faulty's implementation are:
|
1071
|
+
|
1072
|
+
- Rate-based failure thresholds
|
1073
|
+
- Integrated caching inspired by Netflix's [Hystrix][hystrix] with automatic
|
1074
|
+
cache jitter and error fallback.
|
1075
|
+
- Event-based monitoring
|
1076
|
+
- Flexible fault-tolerant storage with optional fallbacks
|
583
1077
|
|
584
|
-
|
585
|
-
|
1078
|
+
Following the principals of the circuit-breaker pattern, the block given to
|
1079
|
+
`run` or `try_run` will always be executed as long as it never raises an error.
|
1080
|
+
If the block _does_ raise an error, then the circuit keeps track of the number
|
1081
|
+
of runs and the failure rate.
|
1082
|
+
|
1083
|
+
Once both thresholds are breached, the circuit is opened. Once open, the
|
1084
|
+
circuit starts the cool-down period. Any executions within that cool-down are
|
1085
|
+
skipped, and a `Faulty::OpenCircuitError` will be raised.
|
1086
|
+
|
1087
|
+
After the cool-down has elapsed, the circuit enters the half-open state. In this
|
1088
|
+
state, Faulty allows a single execution of the block as a test run. If the test
|
1089
|
+
run succeeds, the circuit is fully closed and the circuit state is reset. If the
|
1090
|
+
test run fails, the circuit is opened and the cool-down is reset.
|
1091
|
+
|
1092
|
+
Each time the circuit changes state or executes the block, events are raised
|
1093
|
+
that are sent to the Faulty event notifier. The notifier should be used to track
|
1094
|
+
circuit failure rates, open circuits, etc.
|
1095
|
+
|
1096
|
+
In addition to the classic circuit breaker design, Faulty implements caching
|
1097
|
+
that is integrated with the circuit state. See [Caching](#caching) for more
|
1098
|
+
detail.
|
1099
|
+
|
1100
|
+
### Caching
|
1101
|
+
|
1102
|
+
Faulty integrates caching into it's circuits in a way that is particularly
|
1103
|
+
suited to fault-tolerance. To make use of caching, you must specify the `cache`
|
1104
|
+
configuration option when initializing Faulty or creating a new Faulty instance.
|
1105
|
+
If you're using Rails, this is automatically set to the Rails cache.
|
1106
|
+
|
1107
|
+
Once your cache is configured, you can use the `cache` parameter when running
|
1108
|
+
a circuit to specify a cache key:
|
586
1109
|
|
587
1110
|
```ruby
|
588
|
-
|
589
|
-
|
1111
|
+
feed = Faulty.circuit('rss_feeds')
|
1112
|
+
.try_run(cache: "rss_feeds/#{feed}") do
|
1113
|
+
fetch_feed(feed)
|
1114
|
+
end.or_default([])
|
590
1115
|
```
|
591
1116
|
|
592
|
-
|
593
|
-
|
594
|
-
|
1117
|
+
By default a circuit has the following options:
|
1118
|
+
|
1119
|
+
- `cache_expires_in`: 86400 (1 day). This is sent to the cache backend and
|
1120
|
+
defines how long the cache entry should be stored. After this time elapses,
|
1121
|
+
queries will result in a cache miss.
|
1122
|
+
- `cache_refreshes_after`: 900 (15 minutes). This is used internally by Faulty
|
1123
|
+
to indicate when a cache should be refreshed. It does not affect how long the
|
1124
|
+
cache entry is stored.
|
1125
|
+
- `cache_refresh_jitter`: 180 (3 minutes = 20% of `cache_refreshes_after`). The
|
1126
|
+
maximum number of seconds to randomly add or subtract from
|
1127
|
+
`cache_refreshes_after` when determining whether to refresh a cache entry.
|
1128
|
+
This mitigates the "thundering herd" effect caused by many processes
|
1129
|
+
simultaneously refreshing the cache.
|
1130
|
+
|
1131
|
+
This code will attempt to fetch an RSS feed protected by a circuit. If the feed
|
1132
|
+
is within the cache refresh period, then the result will be returned from the
|
1133
|
+
cache and the block will not be executed regardless of the circuit state.
|
1134
|
+
|
1135
|
+
If the cache is hit, but outside its refresh period, then Faulty will check the
|
1136
|
+
circuit state. If the circuit is closed or half-open, then it will run the
|
1137
|
+
block. If the block is successful, then it will update the circuit, write to the
|
1138
|
+
cache and return the new value.
|
1139
|
+
|
1140
|
+
However, if the cache is hit and the block fails, then that failure is noted
|
1141
|
+
in the circuit and Faulty returns the cached value.
|
1142
|
+
|
1143
|
+
If the circuit is open and the cache is hit, then Faulty will always return the
|
1144
|
+
cached value.
|
1145
|
+
|
1146
|
+
If the cache query results in a miss, then faulty operates as normal. In the
|
1147
|
+
code above, if the circuit is closed, the block will be executed. If the block
|
1148
|
+
succeeds, the cache is refreshed. If the block fails, the default of `[]` will
|
1149
|
+
be returned.
|
1150
|
+
|
1151
|
+
### Fault Tolerance
|
1152
|
+
|
1153
|
+
Faulty backends are fault-tolerant by default. Any `StandardError`s raised by
|
1154
|
+
the storage or cache backends are captured and suppressed. Failure events for
|
1155
|
+
these errors are sent to the notifier.
|
1156
|
+
|
1157
|
+
In case of a flaky storage or cache backend, Faulty also uses independent
|
1158
|
+
in-memory circuits to track failures so that we don't keep calling a backend
|
1159
|
+
that is failing. See the API docs for [`Cache::AutoWire`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/AutoWire),
|
1160
|
+
and [`Storage::AutoWire`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/AutoWire)
|
1161
|
+
for more details.
|
1162
|
+
|
1163
|
+
If the storage backend fails, circuits will default to closed. If the cache
|
1164
|
+
backend fails, all cache queries will miss.
|
595
1165
|
|
596
1166
|
## Implementing a Cache Backend
|
597
1167
|
|
598
1168
|
You can implement your own cache backend by following the documentation in
|
599
|
-
`Faulty::Cache::Interface
|
600
|
-
methods. For example:
|
1169
|
+
[`Faulty::Cache::Interface`](https://www.rubydoc.info/gems/faulty/Faulty/Cache/Interface).
|
1170
|
+
It is a fairly simple API, with only get/set methods. For example:
|
601
1171
|
|
602
1172
|
```ruby
|
603
1173
|
class MyFaultyCache
|
@@ -626,10 +1196,11 @@ users.
|
|
626
1196
|
## Implementing a Storage Backend
|
627
1197
|
|
628
1198
|
You can implement your own storage backend by following the documentation in
|
629
|
-
`Faulty::Storage::Interface
|
630
|
-
|
631
|
-
|
632
|
-
|
1199
|
+
[`Faulty::Storage::Interface`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/Interface).
|
1200
|
+
Since the storage has some tricky requirements regarding concurrency, the
|
1201
|
+
[`Faulty::Storage::Memory`](https://www.rubydoc.info/gems/faulty/Faulty/Storage/Memory)
|
1202
|
+
can be used as a reference implementation. Feel free to open a pull request if
|
1203
|
+
your storage backend would be useful for other users.
|
633
1204
|
|
634
1205
|
## Alternatives
|
635
1206
|
|
@@ -640,8 +1211,8 @@ but there are and have been many other options:
|
|
640
1211
|
|
641
1212
|
- [semian](https://github.com/Shopify/semian): A resiliency toolkit that
|
642
1213
|
includes circuit breakers. It uses adapters to auto-wire circuits, and it has
|
643
|
-
only
|
644
|
-
- [circuitbox](https://github.com/yammer/circuitbox): Similar in
|
1214
|
+
only in-memory storage by design.
|
1215
|
+
- [circuitbox](https://github.com/yammer/circuitbox): Similar in design to
|
645
1216
|
Faulty, but with a different API. It uses Moneta to abstract circuit storage
|
646
1217
|
to allow any key-value store.
|
647
1218
|
|
@@ -660,10 +1231,12 @@ but there are and have been many other options:
|
|
660
1231
|
|
661
1232
|
- Simple API but configurable for advanced users
|
662
1233
|
- Pluggable storage backends (circuitbox also has this)
|
663
|
-
-
|
1234
|
+
- Protected storage access with fallback to safe storage
|
1235
|
+
- Global, or object-oriented configuration with multiple instances
|
664
1236
|
- Integrated caching support tailored for fault-tolerance
|
665
1237
|
- Manually lock circuits open or closed
|
666
1238
|
|
667
1239
|
[api docs]: https://www.rubydoc.info/github/ParentSquare/faulty/master
|
1240
|
+
[michael nygard]: https://www.michaelnygard.com/
|
668
1241
|
[martin fowler]: https://www.martinfowler.com/bliki/CircuitBreaker.html
|
669
1242
|
[hystrix]: https://github.com/Netflix/Hystrix/wiki/How-it-Works
|