resilient 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/README.md +9 -7
- data/examples/basic.rb +81 -0
- data/examples/long_running.rb +42 -0
- data/lib/resilient/circuit_breaker.rb +64 -18
- data/lib/resilient/circuit_breaker/{rolling_config.rb → config.rb} +8 -4
- data/lib/resilient/circuit_breaker/metrics.rb +164 -0
- data/lib/resilient/circuit_breaker/metrics/storage/memory.rb +53 -0
- data/lib/resilient/instrumenters/memory.rb +30 -0
- data/lib/resilient/instrumenters/noop.rb +9 -0
- data/lib/resilient/test/circuit_breaker_interface.rb +30 -0
- data/lib/resilient/test/config_interface.rb +37 -0
- data/lib/resilient/test/metrics_interface.rb +45 -0
- data/lib/resilient/test/metrics_storage_interface.rb +142 -0
- data/lib/resilient/version.rb +1 -1
- metadata +13 -6
- data/TODO.md +0 -4
- data/lib/resilient/circuit_breaker/rolling_metrics.rb +0 -73
- data/lib/resilient/circuit_breaker/rolling_metrics/bucket.rb +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d96ef8b25a0a297da28e4c1f718ab826350a23d
|
4
|
+
data.tar.gz: 8bdf3b03abfedbcf9cd6531a7aad4cd2b3313a18
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 067674abbd47f23741bcc38ca4c99e0ddbcfb187d4e25b84e04841e0055a34ada9e368350868de7fc5363f811e8a20631fcd7d5e13c490918b86242c3f9154f5
|
7
|
+
data.tar.gz: 90c8672939280363a2da23664a4ebadf5e413ce40cf08b38130e00bea1923ac5971c126f2bba5b9c4417e639f16b89e9c92d1b2289e97af3662074e1719c549f
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# Resilient
|
2
2
|
|
3
|
-
Some tools for resilient in ruby.
|
3
|
+
Some tools for resilient in ruby. For now, just a circuit breaker (~~stolen from~~ based on [hystrix](https://github.com/netflix/hystrix)). Soon much more...
|
4
|
+
|
5
|
+
Nothing asynchronous or thread safe yet either.
|
4
6
|
|
5
7
|
## Installation
|
6
8
|
|
@@ -25,13 +27,13 @@ require "resilient/circuit_breaker"
|
|
25
27
|
|
26
28
|
# default config for circuit
|
27
29
|
circuit_breaker = Resilient::CircuitBreaker.new
|
28
|
-
if circuit_breaker.
|
30
|
+
if circuit_breaker.allow_request?
|
29
31
|
begin
|
30
32
|
# do something expensive
|
31
33
|
circuit_breaker.mark_success
|
32
34
|
rescue => boom
|
33
|
-
# do fallback
|
34
35
|
circuit_breaker.mark_failure
|
36
|
+
# do fallback
|
35
37
|
end
|
36
38
|
else
|
37
39
|
# do fallback
|
@@ -41,7 +43,7 @@ end
|
|
41
43
|
customize config of circuit:
|
42
44
|
|
43
45
|
```ruby
|
44
|
-
config = Resilient::CircuitBreaker::
|
46
|
+
config = Resilient::CircuitBreaker::Config.new({
|
45
47
|
# at what percentage of errors should we open the circuit
|
46
48
|
error_threshold_percentage: 50,
|
47
49
|
# do not try request again for 5 seconds
|
@@ -56,7 +58,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
56
58
|
force the circuit to be always open:
|
57
59
|
|
58
60
|
```ruby
|
59
|
-
config = Resilient::CircuitBreaker::
|
61
|
+
config = Resilient::CircuitBreaker::Config.new(force_open: true)
|
60
62
|
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
61
63
|
# etc etc etc
|
62
64
|
```
|
@@ -64,7 +66,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
64
66
|
force the circuit to be always closed:
|
65
67
|
|
66
68
|
```ruby
|
67
|
-
config = Resilient::CircuitBreaker::
|
69
|
+
config = Resilient::CircuitBreaker::Config.new(force_closed: true)
|
68
70
|
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
69
71
|
# etc etc etc
|
70
72
|
```
|
@@ -72,7 +74,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
72
74
|
customize rolling window to be 10 buckets of 1 second each (10 seconds in all):
|
73
75
|
|
74
76
|
```ruby
|
75
|
-
metrics = Resilient::CircuitBreaker::
|
77
|
+
metrics = Resilient::CircuitBreaker::Metrics.new({
|
76
78
|
number_of_buckets: 10,
|
77
79
|
bucket_size_in_seconds: 1,
|
78
80
|
})
|
data/examples/basic.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
# setting up load path
|
2
|
+
require "pathname"
|
3
|
+
root_path = Pathname(__FILE__).dirname.join("..").expand_path
|
4
|
+
lib_path = root_path.join("lib")
|
5
|
+
$:.unshift(lib_path)
|
6
|
+
|
7
|
+
# requiring stuff for this example
|
8
|
+
require "pp"
|
9
|
+
require "resilient/circuit_breaker"
|
10
|
+
|
11
|
+
config = Resilient::CircuitBreaker::Config.new({
|
12
|
+
sleep_window_seconds: 1,
|
13
|
+
request_volume_threshold: 10,
|
14
|
+
error_threshold_percentage: 25,
|
15
|
+
})
|
16
|
+
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
17
|
+
|
18
|
+
# success
|
19
|
+
if circuit_breaker.allow_request?
|
20
|
+
begin
|
21
|
+
puts "do expensive thing"
|
22
|
+
circuit_breaker.mark_success
|
23
|
+
rescue => boom
|
24
|
+
# won't get here in this example
|
25
|
+
circuit_breaker.mark_failure
|
26
|
+
end
|
27
|
+
else
|
28
|
+
raise "will not get here"
|
29
|
+
end
|
30
|
+
|
31
|
+
# failure
|
32
|
+
if circuit_breaker.allow_request?
|
33
|
+
begin
|
34
|
+
raise
|
35
|
+
rescue => boom
|
36
|
+
circuit_breaker.mark_failure
|
37
|
+
puts "failed slow, do fallback"
|
38
|
+
end
|
39
|
+
else
|
40
|
+
raise "will not get here"
|
41
|
+
end
|
42
|
+
|
43
|
+
# trip circuit, imagine this being same as above but in real life...
|
44
|
+
# also, we have to fail at least the request volume threshold number of times
|
45
|
+
circuit_breaker.config.request_volume_threshold.times do
|
46
|
+
circuit_breaker.mark_failure
|
47
|
+
end
|
48
|
+
|
49
|
+
# fail fast
|
50
|
+
if circuit_breaker.allow_request?
|
51
|
+
raise "will not get here"
|
52
|
+
else
|
53
|
+
puts "failed fast, do fallback"
|
54
|
+
end
|
55
|
+
|
56
|
+
now = Time.now
|
57
|
+
|
58
|
+
while Time.now - now < 3
|
59
|
+
if circuit_breaker.allow_request?
|
60
|
+
puts "doing a single attempt as we've failed fast for sleep_window_seconds"
|
61
|
+
break
|
62
|
+
else
|
63
|
+
puts "failed fast, do fallback"
|
64
|
+
end
|
65
|
+
sleep rand(0.1)
|
66
|
+
end
|
67
|
+
|
68
|
+
if circuit_breaker.allow_request?
|
69
|
+
raise "will not get here"
|
70
|
+
else
|
71
|
+
puts "request denied because single request has not been marked success yet"
|
72
|
+
end
|
73
|
+
|
74
|
+
puts "marking single request as success"
|
75
|
+
circuit_breaker.mark_success
|
76
|
+
|
77
|
+
if circuit_breaker.allow_request?
|
78
|
+
puts "circuit reset and back closed now, allowing requests"
|
79
|
+
else
|
80
|
+
raise "will not get here"
|
81
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# setting up load path
|
2
|
+
require "pathname"
|
3
|
+
root_path = Pathname(__FILE__).dirname.join("..").expand_path
|
4
|
+
lib_path = root_path.join("lib")
|
5
|
+
$:.unshift(lib_path)
|
6
|
+
|
7
|
+
# requiring stuff for this example
|
8
|
+
require "pp"
|
9
|
+
require "resilient/circuit_breaker"
|
10
|
+
|
11
|
+
config = Resilient::CircuitBreaker::Config.new({
|
12
|
+
sleep_window_seconds: 5,
|
13
|
+
request_volume_threshold: 20,
|
14
|
+
error_threshold_percentage: 10,
|
15
|
+
number_of_buckets: 10,
|
16
|
+
bucket_size_in_seconds: 1,
|
17
|
+
})
|
18
|
+
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
19
|
+
|
20
|
+
iterations = 0
|
21
|
+
loop do
|
22
|
+
if circuit_breaker.allow_request?
|
23
|
+
begin
|
24
|
+
puts "request allowed"
|
25
|
+
raise if rand(100) < 10
|
26
|
+
puts "request succeeded"
|
27
|
+
circuit_breaker.mark_success
|
28
|
+
rescue => boom
|
29
|
+
puts "request failed"
|
30
|
+
circuit_breaker.mark_failure
|
31
|
+
end
|
32
|
+
else
|
33
|
+
puts "request denied"
|
34
|
+
end
|
35
|
+
puts "\n"
|
36
|
+
sleep 0.1
|
37
|
+
iterations += 1
|
38
|
+
|
39
|
+
if iterations % 10 == 0
|
40
|
+
p successes: circuit_breaker.metrics.successes, failures: circuit_breaker.metrics.failures, error_percentage: circuit_breaker.metrics.error_percentage, buckets: circuit_breaker.metrics.buckets.length
|
41
|
+
end
|
42
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require "resilient/circuit_breaker/
|
2
|
-
require "resilient/circuit_breaker/
|
1
|
+
require "resilient/circuit_breaker/metrics"
|
2
|
+
require "resilient/circuit_breaker/config"
|
3
3
|
|
4
4
|
module Resilient
|
5
5
|
class CircuitBreaker
|
@@ -8,14 +8,14 @@ module Resilient
|
|
8
8
|
attr_reader :open
|
9
9
|
attr_reader :opened_or_last_checked_at_epoch
|
10
10
|
|
11
|
-
def initialize(open: false, config:
|
11
|
+
def initialize(open: false, config: Config.new, metrics: Metrics.new)
|
12
12
|
@open = open
|
13
13
|
@opened_or_last_checked_at_epoch = 0
|
14
14
|
@config = config
|
15
15
|
@metrics = if metrics
|
16
16
|
metrics
|
17
17
|
else
|
18
|
-
|
18
|
+
Metrics.new({
|
19
19
|
number_of_buckets: config.number_of_buckets,
|
20
20
|
bucket_size_in_seconds: config.bucket_size_in_seconds,
|
21
21
|
})
|
@@ -23,32 +23,74 @@ module Resilient
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def allow_request?
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
26
|
+
default_payload = {
|
27
|
+
force_open: false,
|
28
|
+
force_closed: false,
|
29
|
+
}
|
30
|
+
|
31
|
+
instrument("resilient.circuit_breaker.allow_request", default_payload) { |payload|
|
32
|
+
result = if payload[:force_open] = @config.force_open
|
33
|
+
false
|
34
|
+
else
|
35
|
+
if payload[:force_closed] = @config.force_closed
|
36
|
+
# we still want to simulate normal behavior/metrics like open, allow
|
37
|
+
# single request, etc. so it is possible to test config in
|
38
|
+
# production without impact
|
39
|
+
if payload[:open] = open?
|
40
|
+
payload[:allow_single_request] = allow_single_request?
|
41
|
+
end
|
42
|
+
|
43
|
+
true
|
44
|
+
else
|
45
|
+
if !(payload[:open] = open?)
|
46
|
+
true
|
47
|
+
else
|
48
|
+
payload[:allow_single_request] = allow_single_request?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
payload[:result] = result
|
54
|
+
}
|
30
55
|
end
|
31
56
|
|
32
57
|
def mark_success
|
33
|
-
|
58
|
+
default_payload = {
|
59
|
+
closed_the_circuit: false,
|
60
|
+
}
|
61
|
+
|
62
|
+
instrument("resilient.circuit_breaker.mark_success", default_payload) { |payload|
|
63
|
+
if @open
|
64
|
+
payload[:closed_the_circuit] = true
|
65
|
+
close_circuit
|
66
|
+
else
|
67
|
+
@metrics.mark_success
|
68
|
+
end
|
69
|
+
nil
|
70
|
+
}
|
34
71
|
end
|
35
72
|
|
36
73
|
def mark_failure
|
37
|
-
|
74
|
+
instrument("resilient.circuit_breaker.mark_failure") { |payload|
|
75
|
+
@metrics.mark_failure
|
76
|
+
nil
|
77
|
+
}
|
38
78
|
end
|
39
79
|
|
40
80
|
def reset
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
81
|
+
instrument("resilient.circuit_breaker.reset") { |payload|
|
82
|
+
@open = false
|
83
|
+
@opened_or_last_checked_at_epoch = 0
|
84
|
+
@metrics.reset
|
85
|
+
nil
|
86
|
+
}
|
45
87
|
end
|
46
88
|
|
47
89
|
private
|
48
90
|
|
49
91
|
def open_circuit
|
50
|
-
@open = true
|
51
92
|
@opened_or_last_checked_at_epoch = Time.now.to_i
|
93
|
+
@open = true
|
52
94
|
end
|
53
95
|
|
54
96
|
def close_circuit
|
@@ -71,6 +113,7 @@ module Resilient
|
|
71
113
|
return false if under_error_threshold_percentage?
|
72
114
|
|
73
115
|
open_circuit
|
116
|
+
true
|
74
117
|
end
|
75
118
|
|
76
119
|
def closed?
|
@@ -78,15 +121,18 @@ module Resilient
|
|
78
121
|
end
|
79
122
|
|
80
123
|
def allow_single_request?
|
81
|
-
try_next_request_at = @opened_or_last_checked_at_epoch + @config.sleep_window_seconds
|
82
124
|
now = Time.now.to_i
|
83
125
|
|
84
|
-
if @open && now >
|
85
|
-
@opened_or_last_checked_at_epoch = now
|
126
|
+
if @open && now > (@opened_or_last_checked_at_epoch + @config.sleep_window_seconds)
|
127
|
+
@opened_or_last_checked_at_epoch = now
|
86
128
|
true
|
87
129
|
else
|
88
130
|
false
|
89
131
|
end
|
90
132
|
end
|
133
|
+
|
134
|
+
def instrument(name, payload = {}, &block)
|
135
|
+
config.instrumenter.instrument(name, payload, &block)
|
136
|
+
end
|
91
137
|
end
|
92
138
|
end
|
@@ -1,17 +1,21 @@
|
|
1
|
+
require "resilient/instrumenters/noop"
|
2
|
+
|
1
3
|
module Resilient
|
2
4
|
class CircuitBreaker
|
3
|
-
class
|
4
|
-
attr_reader :error_threshold_percentage
|
5
|
-
attr_reader :sleep_window_seconds
|
6
|
-
attr_reader :request_volume_threshold
|
5
|
+
class Config
|
7
6
|
attr_reader :force_open
|
8
7
|
attr_reader :force_closed
|
8
|
+
attr_reader :instrumenter
|
9
|
+
attr_reader :sleep_window_seconds
|
10
|
+
attr_reader :request_volume_threshold
|
11
|
+
attr_reader :error_threshold_percentage
|
9
12
|
attr_reader :number_of_buckets
|
10
13
|
attr_reader :bucket_size_in_seconds
|
11
14
|
|
12
15
|
def initialize(options = {})
|
13
16
|
@force_open = options.fetch(:force_open, false)
|
14
17
|
@force_closed = options.fetch(:force_closed, false)
|
18
|
+
@instrumenter = options.fetch(:instrumenter, Instrumenters::Noop)
|
15
19
|
@sleep_window_seconds = options.fetch(:sleep_window_seconds, 5)
|
16
20
|
@request_volume_threshold = options.fetch(:request_volume_threshold, 20)
|
17
21
|
@error_threshold_percentage = options.fetch(:error_threshold_percentage, 50)
|
@@ -0,0 +1,164 @@
|
|
1
|
+
require "resilient/circuit_breaker/metrics/storage/memory"
|
2
|
+
|
3
|
+
module Resilient
|
4
|
+
class CircuitBreaker
|
5
|
+
class Metrics
|
6
|
+
attr_reader :number_of_buckets
|
7
|
+
attr_reader :bucket_size_in_seconds
|
8
|
+
attr_reader :buckets
|
9
|
+
attr_reader :storage
|
10
|
+
|
11
|
+
StorageSuccessKeys = [
|
12
|
+
:successes,
|
13
|
+
].freeze
|
14
|
+
|
15
|
+
StorageFailureKeys = [
|
16
|
+
:failures,
|
17
|
+
].freeze
|
18
|
+
|
19
|
+
StorageKeys = (StorageSuccessKeys + StorageFailureKeys).freeze
|
20
|
+
|
21
|
+
class Bucket
|
22
|
+
attr_reader :timestamp_start
|
23
|
+
attr_reader :timestamp_end
|
24
|
+
|
25
|
+
def initialize(timestamp_start, timestamp_end)
|
26
|
+
@timestamp_start = timestamp_start
|
27
|
+
@timestamp_end = timestamp_end
|
28
|
+
end
|
29
|
+
|
30
|
+
def prune_before(number_of_buckets, bucket_size)
|
31
|
+
@timestamp_end - (number_of_buckets * bucket_size.seconds)
|
32
|
+
end
|
33
|
+
|
34
|
+
def include?(timestamp)
|
35
|
+
timestamp >= @timestamp_start && timestamp <= @timestamp_end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class BucketRange
|
40
|
+
def self.generate(timestamp, number_of_buckets, bucket_size)
|
41
|
+
end_bucket = bucket_size.bucket(timestamp)
|
42
|
+
start_bucket = bucket_size.bucket(end_bucket.prune_before(number_of_buckets, bucket_size))
|
43
|
+
bucket_range = new(start_bucket, end_bucket)
|
44
|
+
end
|
45
|
+
|
46
|
+
attr_reader :start_bucket
|
47
|
+
attr_reader :end_bucket
|
48
|
+
|
49
|
+
def initialize(start_bucket, end_bucket)
|
50
|
+
@start_bucket = start_bucket
|
51
|
+
@end_bucket = end_bucket
|
52
|
+
end
|
53
|
+
|
54
|
+
def prune?(bucket)
|
55
|
+
bucket.timestamp_end <= @start_bucket.timestamp_end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class BucketSize
|
60
|
+
attr_reader :seconds
|
61
|
+
|
62
|
+
def initialize(seconds)
|
63
|
+
@seconds = seconds
|
64
|
+
end
|
65
|
+
|
66
|
+
def aligned_start(timestamp = Time.now.to_i)
|
67
|
+
timestamp / @seconds * @seconds
|
68
|
+
end
|
69
|
+
|
70
|
+
def aligned_end(timestamp = Time.now.to_i)
|
71
|
+
aligned_start(timestamp) + @seconds - 1
|
72
|
+
end
|
73
|
+
|
74
|
+
def bucket(timestamp = Time.now.to_i)
|
75
|
+
Bucket.new aligned_start(timestamp), aligned_end(timestamp)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def initialize(options = {})
|
80
|
+
@number_of_buckets = options.fetch(:number_of_buckets, 6)
|
81
|
+
@bucket_size = BucketSize.new(options.fetch(:bucket_size_in_seconds, 10))
|
82
|
+
@storage = options.fetch(:storage) { Storage::Memory.new }
|
83
|
+
@buckets = []
|
84
|
+
end
|
85
|
+
|
86
|
+
def mark_success
|
87
|
+
@storage.increment(current_bucket, StorageSuccessKeys)
|
88
|
+
prune_buckets
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
def mark_failure
|
93
|
+
@storage.increment(current_bucket, StorageFailureKeys)
|
94
|
+
prune_buckets
|
95
|
+
nil
|
96
|
+
end
|
97
|
+
|
98
|
+
def successes
|
99
|
+
prune_buckets
|
100
|
+
@storage.sum(@buckets, :successes)[:successes]
|
101
|
+
end
|
102
|
+
|
103
|
+
def failures
|
104
|
+
prune_buckets
|
105
|
+
@storage.sum(@buckets, :failures)[:failures]
|
106
|
+
end
|
107
|
+
|
108
|
+
def requests
|
109
|
+
prune_buckets
|
110
|
+
requests = 0
|
111
|
+
@storage.sum(@buckets, StorageKeys).each do |key, value|
|
112
|
+
requests += value
|
113
|
+
end
|
114
|
+
requests
|
115
|
+
end
|
116
|
+
|
117
|
+
def error_percentage
|
118
|
+
prune_buckets
|
119
|
+
|
120
|
+
result = @storage.sum(@buckets, StorageKeys)
|
121
|
+
successes = result[:successes]
|
122
|
+
failures = result[:failures]
|
123
|
+
|
124
|
+
requests = successes + failures
|
125
|
+
return 0 if failures == 0 || requests == 0
|
126
|
+
|
127
|
+
((failures / requests.to_f) * 100).round
|
128
|
+
end
|
129
|
+
|
130
|
+
def reset
|
131
|
+
@storage.reset(@buckets, StorageKeys)
|
132
|
+
nil
|
133
|
+
end
|
134
|
+
|
135
|
+
private
|
136
|
+
|
137
|
+
def current_bucket(timestamp = Time.now.to_i)
|
138
|
+
bucket = @buckets.detect { |bucket| bucket.include?(timestamp) }
|
139
|
+
return bucket if bucket
|
140
|
+
|
141
|
+
bucket = @bucket_size.bucket(timestamp)
|
142
|
+
@buckets.push bucket
|
143
|
+
|
144
|
+
bucket
|
145
|
+
end
|
146
|
+
|
147
|
+
def prune_buckets(timestamp = Time.now.to_i)
|
148
|
+
pruned_buckets = []
|
149
|
+
bucket_range = BucketRange.generate(timestamp, @number_of_buckets, @bucket_size)
|
150
|
+
|
151
|
+
@buckets.delete_if { |bucket|
|
152
|
+
if bucket_range.prune?(bucket)
|
153
|
+
pruned_buckets << bucket
|
154
|
+
true
|
155
|
+
end
|
156
|
+
}
|
157
|
+
|
158
|
+
if pruned_buckets.any?
|
159
|
+
@storage.prune(pruned_buckets, StorageKeys)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Resilient
|
2
|
+
class CircuitBreaker
|
3
|
+
class Metrics
|
4
|
+
module Storage
|
5
|
+
class Memory
|
6
|
+
attr_reader :source
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@source = Hash.new { |h, k| h[k] = Hash.new(0) }
|
10
|
+
end
|
11
|
+
|
12
|
+
def increment(buckets, keys)
|
13
|
+
Array(buckets).each do |bucket|
|
14
|
+
Array(keys).each do |key|
|
15
|
+
@source[bucket][key] += 1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def get(buckets, keys)
|
21
|
+
result = Hash.new { |h, k| h[k] = Hash.new(0) }
|
22
|
+
Array(buckets).each do |bucket|
|
23
|
+
Array(keys).each do |key|
|
24
|
+
result[bucket][key] = @source[bucket][key]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
result
|
28
|
+
end
|
29
|
+
|
30
|
+
def sum(buckets, keys)
|
31
|
+
response = Hash.new(0)
|
32
|
+
Array(buckets).each do |bucket|
|
33
|
+
Array(keys).each do |key|
|
34
|
+
response[key] += @source[bucket][key]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
response
|
38
|
+
end
|
39
|
+
|
40
|
+
def reset(buckets, keys)
|
41
|
+
Array(buckets).each do |bucket|
|
42
|
+
@source.delete(bucket)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def prune(buckets, keys)
|
47
|
+
reset(buckets, keys)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Resilient
|
2
|
+
module Instrumenters
|
3
|
+
# Instrumentor that is useful for tests as it stores each of the events that
|
4
|
+
# are instrumented.
|
5
|
+
class Memory
|
6
|
+
Event = Struct.new(:name, :payload, :result)
|
7
|
+
|
8
|
+
attr_reader :events
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@events = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def instrument(name, payload = {})
|
15
|
+
# Copy the payload to guard against later modifications to it, and to
|
16
|
+
# ensure that all instrumentation code uses the payload passed to the
|
17
|
+
# block rather than the one passed to #instrument.
|
18
|
+
payload = payload.dup
|
19
|
+
|
20
|
+
result = if block_given?
|
21
|
+
yield payload
|
22
|
+
else
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
@events << Event.new(name, payload, result)
|
26
|
+
result
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module CircuitBreakerInterface
|
4
|
+
def test_responds_to_allow_request
|
5
|
+
assert_respond_to @object, :allow_request?
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_mark_success
|
9
|
+
assert_respond_to @object, :mark_success
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_mark_success_returns_nothing
|
13
|
+
assert_nil @object.mark_success
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_mark_failure
|
17
|
+
assert_respond_to @object, :mark_failure
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_mark_failure_returns_nothing
|
21
|
+
assert_nil @object.mark_failure
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_reset
|
25
|
+
assert_respond_to @object, :reset
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module ConfigInterface
|
4
|
+
def test_responds_to_force_open
|
5
|
+
assert_respond_to @object, :force_open
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_force_closed
|
9
|
+
assert_respond_to @object, :force_closed
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_instrumenter
|
13
|
+
assert_respond_to @object, :instrumenter
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_sleep_window_seconds
|
17
|
+
assert_respond_to @object, :sleep_window_seconds
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_request_volume_threshold
|
21
|
+
assert_respond_to @object, :request_volume_threshold
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_error_threshold_percentage
|
25
|
+
assert_respond_to @object, :error_threshold_percentage
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_responds_to_number_of_buckets
|
29
|
+
assert_respond_to @object, :number_of_buckets
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_responds_to_bucket_size_in_seconds
|
33
|
+
assert_respond_to @object, :bucket_size_in_seconds
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module MetricsInterface
|
4
|
+
def test_responds_to_mark_success
|
5
|
+
assert_respond_to @object, :mark_success
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_mark_success_returns_nothing
|
9
|
+
assert_nil @object.mark_success
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_mark_failure
|
13
|
+
assert_respond_to @object, :mark_failure
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_mark_failure_returns_nothing
|
17
|
+
assert_nil @object.mark_failure
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_successes
|
21
|
+
assert_respond_to @object, :successes
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_failures
|
25
|
+
assert_respond_to @object, :failures
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_responds_to_requests
|
29
|
+
assert_respond_to @object, :requests
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_responds_to_error_percentage
|
33
|
+
assert_respond_to @object, :error_percentage
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_responds_to_reset
|
37
|
+
assert_respond_to @object, :reset
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_reset_returns_nothing
|
41
|
+
assert_nil @object.reset
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module MetricsStorageInterface
|
4
|
+
def test_responds_to_increment
|
5
|
+
assert_respond_to @object, :increment
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_get
|
9
|
+
assert_respond_to @object, :get
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_sum
|
13
|
+
assert_respond_to @object, :sum
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_prune
|
17
|
+
assert_respond_to @object, :prune
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_reset
|
21
|
+
assert_respond_to @object, :reset
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_increment
|
25
|
+
buckets = [
|
26
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
27
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
28
|
+
]
|
29
|
+
keys = [
|
30
|
+
:successes,
|
31
|
+
:failures,
|
32
|
+
]
|
33
|
+
@object.increment(buckets, keys)
|
34
|
+
assert_equal 1, @object.source[buckets[0]][:successes]
|
35
|
+
assert_equal 1, @object.source[buckets[0]][:failures]
|
36
|
+
assert_equal 1, @object.source[buckets[1]][:successes]
|
37
|
+
assert_equal 1, @object.source[buckets[1]][:failures]
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_get_defaults
|
41
|
+
buckets = [
|
42
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
43
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
44
|
+
]
|
45
|
+
keys = [
|
46
|
+
:successes,
|
47
|
+
:failures,
|
48
|
+
]
|
49
|
+
result = @object.get(buckets, keys)
|
50
|
+
assert_equal 0, result[buckets[0]][:successes]
|
51
|
+
assert_equal 0, result[buckets[0]][:failures]
|
52
|
+
assert_equal 0, result[buckets[1]][:successes]
|
53
|
+
assert_equal 0, result[buckets[1]][:failures]
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_get_with_values
|
57
|
+
buckets = [
|
58
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
59
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
60
|
+
]
|
61
|
+
keys = [
|
62
|
+
:successes,
|
63
|
+
:failures,
|
64
|
+
]
|
65
|
+
@object.increment(buckets, keys)
|
66
|
+
@object.increment(buckets, keys)
|
67
|
+
@object.increment(buckets[0], keys)
|
68
|
+
result = @object.get(buckets, keys)
|
69
|
+
assert_equal 3, result[buckets[0]][:successes]
|
70
|
+
assert_equal 3, result[buckets[0]][:failures]
|
71
|
+
assert_equal 2, result[buckets[1]][:successes]
|
72
|
+
assert_equal 2, result[buckets[1]][:failures]
|
73
|
+
end
|
74
|
+
|
75
|
+
def test_sum
|
76
|
+
buckets = [
|
77
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
78
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
79
|
+
]
|
80
|
+
keys = [
|
81
|
+
:successes,
|
82
|
+
:failures,
|
83
|
+
]
|
84
|
+
@object.increment(buckets, keys)
|
85
|
+
@object.increment(buckets, keys)
|
86
|
+
@object.increment(buckets[0], keys)
|
87
|
+
|
88
|
+
assert_equal 5, @object.sum(buckets, [:successes])[:successes]
|
89
|
+
assert_equal 5, @object.sum(buckets, [:failures])[:failures]
|
90
|
+
assert_equal 10, @object.sum(buckets, keys).values.inject(0) { |sum, value| sum += value }
|
91
|
+
|
92
|
+
assert_equal 3, @object.sum(buckets[0], [:successes])[:successes]
|
93
|
+
assert_equal 3, @object.sum(buckets[0], [:failures])[:failures]
|
94
|
+
assert_equal 6, @object.sum(buckets[0], keys).values.inject(0) { |sum, value| sum += value }
|
95
|
+
|
96
|
+
assert_equal 2, @object.sum(buckets[1], [:successes])[:successes]
|
97
|
+
assert_equal 2, @object.sum(buckets[1], [:failures])[:failures]
|
98
|
+
assert_equal 4, @object.sum(buckets[1], keys).values.inject(0) { |sum, value| sum += value }
|
99
|
+
end
|
100
|
+
|
101
|
+
def test_reset
|
102
|
+
buckets = [
|
103
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
104
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
105
|
+
]
|
106
|
+
keys = [
|
107
|
+
:successes,
|
108
|
+
:failures,
|
109
|
+
]
|
110
|
+
@object.increment(buckets, keys)
|
111
|
+
@object.increment(buckets, keys)
|
112
|
+
@object.increment(buckets[0], keys)
|
113
|
+
@object.reset(buckets, keys)
|
114
|
+
result = @object.get(buckets, keys)
|
115
|
+
assert_equal 0, result[buckets[0]][:successes]
|
116
|
+
assert_equal 0, result[buckets[0]][:failures]
|
117
|
+
assert_equal 0, result[buckets[1]][:successes]
|
118
|
+
assert_equal 0, result[buckets[1]][:failures]
|
119
|
+
end
|
120
|
+
|
121
|
+
def test_prune
|
122
|
+
buckets = [
|
123
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
124
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
125
|
+
]
|
126
|
+
keys = [
|
127
|
+
:successes,
|
128
|
+
:failures,
|
129
|
+
]
|
130
|
+
@object.increment(buckets, keys)
|
131
|
+
@object.increment(buckets, keys)
|
132
|
+
@object.increment(buckets[0], keys)
|
133
|
+
@object.prune(buckets, keys)
|
134
|
+
result = @object.get(buckets, keys)
|
135
|
+
assert_equal 0, result[buckets[0]][:successes]
|
136
|
+
assert_equal 0, result[buckets[0]][:failures]
|
137
|
+
assert_equal 0, result[buckets[1]][:successes]
|
138
|
+
assert_equal 0, result[buckets[1]][:failures]
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
data/lib/resilient/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resilient
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Nunemaker
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-12-
|
11
|
+
date: 2015-12-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,12 +66,19 @@ files:
|
|
66
66
|
- LICENSE.txt
|
67
67
|
- README.md
|
68
68
|
- Rakefile
|
69
|
-
-
|
69
|
+
- examples/basic.rb
|
70
|
+
- examples/long_running.rb
|
70
71
|
- lib/resilient.rb
|
71
72
|
- lib/resilient/circuit_breaker.rb
|
72
|
-
- lib/resilient/circuit_breaker/
|
73
|
-
- lib/resilient/circuit_breaker/
|
74
|
-
- lib/resilient/circuit_breaker/
|
73
|
+
- lib/resilient/circuit_breaker/config.rb
|
74
|
+
- lib/resilient/circuit_breaker/metrics.rb
|
75
|
+
- lib/resilient/circuit_breaker/metrics/storage/memory.rb
|
76
|
+
- lib/resilient/instrumenters/memory.rb
|
77
|
+
- lib/resilient/instrumenters/noop.rb
|
78
|
+
- lib/resilient/test/circuit_breaker_interface.rb
|
79
|
+
- lib/resilient/test/config_interface.rb
|
80
|
+
- lib/resilient/test/metrics_interface.rb
|
81
|
+
- lib/resilient/test/metrics_storage_interface.rb
|
75
82
|
- lib/resilient/version.rb
|
76
83
|
- resilient.gemspec
|
77
84
|
- script/bootstrap
|
data/TODO.md
DELETED
@@ -1,4 +0,0 @@
|
|
1
|
-
- [ ] instrumentation
|
2
|
-
- [ ] add duration to mark success/failure; use it for tracking latency and for making circuit decisions
|
3
|
-
- [ ] add timeout to metrics separate from failures (if we add duration)?
|
4
|
-
- [ ] should force closed still instrument and all that so we can test in prod before enabling or should we allow enabling/disabling with instrumentation in a different way?
|
@@ -1,73 +0,0 @@
|
|
1
|
-
require "resilient/circuit_breaker/rolling_metrics/bucket"
|
2
|
-
|
3
|
-
module Resilient
|
4
|
-
class CircuitBreaker
|
5
|
-
class RollingMetrics
|
6
|
-
attr_reader :number_of_buckets
|
7
|
-
attr_reader :bucket_size_in_seconds
|
8
|
-
attr_reader :buckets
|
9
|
-
|
10
|
-
def initialize(options = {})
|
11
|
-
@number_of_buckets = options.fetch(:number_of_buckets, 6)
|
12
|
-
@bucket_size_in_seconds = options.fetch(:bucket_size_in_seconds, 10)
|
13
|
-
reset
|
14
|
-
end
|
15
|
-
|
16
|
-
def mark_success
|
17
|
-
timestamp = Time.now.to_i
|
18
|
-
bucket(timestamp).mark_success
|
19
|
-
prune_buckets(timestamp)
|
20
|
-
nil
|
21
|
-
end
|
22
|
-
|
23
|
-
def mark_failure
|
24
|
-
timestamp = Time.now.to_i
|
25
|
-
bucket(timestamp).mark_failure
|
26
|
-
prune_buckets(timestamp)
|
27
|
-
nil
|
28
|
-
end
|
29
|
-
|
30
|
-
def successes
|
31
|
-
prune_buckets
|
32
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.successes }
|
33
|
-
end
|
34
|
-
|
35
|
-
def failures
|
36
|
-
prune_buckets
|
37
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.failures }
|
38
|
-
end
|
39
|
-
|
40
|
-
def requests
|
41
|
-
prune_buckets
|
42
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.requests }
|
43
|
-
end
|
44
|
-
|
45
|
-
def error_percentage
|
46
|
-
return 0 if failures == 0 || requests == 0
|
47
|
-
((failures / requests.to_f) * 100).to_i
|
48
|
-
end
|
49
|
-
|
50
|
-
def reset
|
51
|
-
@buckets = []
|
52
|
-
nil
|
53
|
-
end
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def bucket(timestamp)
|
58
|
-
bucket = @buckets.detect { |bucket| bucket.include?(timestamp) }
|
59
|
-
return bucket if bucket
|
60
|
-
|
61
|
-
bucket = Bucket.new(timestamp, timestamp + @bucket_size_in_seconds - 1)
|
62
|
-
@buckets.push bucket
|
63
|
-
|
64
|
-
bucket
|
65
|
-
end
|
66
|
-
|
67
|
-
def prune_buckets(timestamp = Time.now.to_i)
|
68
|
-
cutoff = timestamp - (@number_of_buckets * @bucket_size_in_seconds)
|
69
|
-
@buckets.delete_if { |bucket| bucket.prune?(cutoff) }
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
module Resilient
|
2
|
-
class CircuitBreaker
|
3
|
-
class RollingMetrics
|
4
|
-
class Bucket
|
5
|
-
attr_reader :successes
|
6
|
-
attr_reader :failures
|
7
|
-
|
8
|
-
def initialize(timestamp_start, timestamp_end)
|
9
|
-
@timestamp_start = timestamp_start
|
10
|
-
@timestamp_end = timestamp_end
|
11
|
-
@successes = 0
|
12
|
-
@failures = 0
|
13
|
-
end
|
14
|
-
|
15
|
-
def mark_success
|
16
|
-
@successes += 1
|
17
|
-
end
|
18
|
-
|
19
|
-
def mark_failure
|
20
|
-
@failures += 1
|
21
|
-
end
|
22
|
-
|
23
|
-
def requests
|
24
|
-
@successes + @failures
|
25
|
-
end
|
26
|
-
|
27
|
-
def include?(timestamp)
|
28
|
-
timestamp >= @timestamp_start && timestamp <= @timestamp_end
|
29
|
-
end
|
30
|
-
|
31
|
-
def prune?(timestamp)
|
32
|
-
@timestamp_end <= timestamp
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|