resilient 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/README.md +9 -7
- data/examples/basic.rb +81 -0
- data/examples/long_running.rb +42 -0
- data/lib/resilient/circuit_breaker.rb +64 -18
- data/lib/resilient/circuit_breaker/{rolling_config.rb → config.rb} +8 -4
- data/lib/resilient/circuit_breaker/metrics.rb +164 -0
- data/lib/resilient/circuit_breaker/metrics/storage/memory.rb +53 -0
- data/lib/resilient/instrumenters/memory.rb +30 -0
- data/lib/resilient/instrumenters/noop.rb +9 -0
- data/lib/resilient/test/circuit_breaker_interface.rb +30 -0
- data/lib/resilient/test/config_interface.rb +37 -0
- data/lib/resilient/test/metrics_interface.rb +45 -0
- data/lib/resilient/test/metrics_storage_interface.rb +142 -0
- data/lib/resilient/version.rb +1 -1
- metadata +13 -6
- data/TODO.md +0 -4
- data/lib/resilient/circuit_breaker/rolling_metrics.rb +0 -73
- data/lib/resilient/circuit_breaker/rolling_metrics/bucket.rb +0 -37
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d96ef8b25a0a297da28e4c1f718ab826350a23d
|
4
|
+
data.tar.gz: 8bdf3b03abfedbcf9cd6531a7aad4cd2b3313a18
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 067674abbd47f23741bcc38ca4c99e0ddbcfb187d4e25b84e04841e0055a34ada9e368350868de7fc5363f811e8a20631fcd7d5e13c490918b86242c3f9154f5
|
7
|
+
data.tar.gz: 90c8672939280363a2da23664a4ebadf5e413ce40cf08b38130e00bea1923ac5971c126f2bba5b9c4417e639f16b89e9c92d1b2289e97af3662074e1719c549f
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# Resilient
|
2
2
|
|
3
|
-
Some tools for resilient in ruby.
|
3
|
+
Some tools for resilient in ruby. For now, just a circuit breaker (~~stolen from~~ based on [hystrix](https://github.com/netflix/hystrix)). Soon much more...
|
4
|
+
|
5
|
+
Nothing asynchronous or thread safe yet either.
|
4
6
|
|
5
7
|
## Installation
|
6
8
|
|
@@ -25,13 +27,13 @@ require "resilient/circuit_breaker"
|
|
25
27
|
|
26
28
|
# default config for circuit
|
27
29
|
circuit_breaker = Resilient::CircuitBreaker.new
|
28
|
-
if circuit_breaker.
|
30
|
+
if circuit_breaker.allow_request?
|
29
31
|
begin
|
30
32
|
# do something expensive
|
31
33
|
circuit_breaker.mark_success
|
32
34
|
rescue => boom
|
33
|
-
# do fallback
|
34
35
|
circuit_breaker.mark_failure
|
36
|
+
# do fallback
|
35
37
|
end
|
36
38
|
else
|
37
39
|
# do fallback
|
@@ -41,7 +43,7 @@ end
|
|
41
43
|
customize config of circuit:
|
42
44
|
|
43
45
|
```ruby
|
44
|
-
config = Resilient::CircuitBreaker::
|
46
|
+
config = Resilient::CircuitBreaker::Config.new({
|
45
47
|
# at what percentage of errors should we open the circuit
|
46
48
|
error_threshold_percentage: 50,
|
47
49
|
# do not try request again for 5 seconds
|
@@ -56,7 +58,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
56
58
|
force the circuit to be always open:
|
57
59
|
|
58
60
|
```ruby
|
59
|
-
config = Resilient::CircuitBreaker::
|
61
|
+
config = Resilient::CircuitBreaker::Config.new(force_open: true)
|
60
62
|
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
61
63
|
# etc etc etc
|
62
64
|
```
|
@@ -64,7 +66,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
64
66
|
force the circuit to be always closed:
|
65
67
|
|
66
68
|
```ruby
|
67
|
-
config = Resilient::CircuitBreaker::
|
69
|
+
config = Resilient::CircuitBreaker::Config.new(force_closed: true)
|
68
70
|
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
69
71
|
# etc etc etc
|
70
72
|
```
|
@@ -72,7 +74,7 @@ circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
|
72
74
|
customize rolling window to be 10 buckets of 1 second each (10 seconds in all):
|
73
75
|
|
74
76
|
```ruby
|
75
|
-
metrics = Resilient::CircuitBreaker::
|
77
|
+
metrics = Resilient::CircuitBreaker::Metrics.new({
|
76
78
|
number_of_buckets: 10,
|
77
79
|
bucket_size_in_seconds: 1,
|
78
80
|
})
|
data/examples/basic.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
# setting up load path
|
2
|
+
require "pathname"
|
3
|
+
root_path = Pathname(__FILE__).dirname.join("..").expand_path
|
4
|
+
lib_path = root_path.join("lib")
|
5
|
+
$:.unshift(lib_path)
|
6
|
+
|
7
|
+
# requiring stuff for this example
|
8
|
+
require "pp"
|
9
|
+
require "resilient/circuit_breaker"
|
10
|
+
|
11
|
+
config = Resilient::CircuitBreaker::Config.new({
|
12
|
+
sleep_window_seconds: 1,
|
13
|
+
request_volume_threshold: 10,
|
14
|
+
error_threshold_percentage: 25,
|
15
|
+
})
|
16
|
+
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
17
|
+
|
18
|
+
# success
|
19
|
+
if circuit_breaker.allow_request?
|
20
|
+
begin
|
21
|
+
puts "do expensive thing"
|
22
|
+
circuit_breaker.mark_success
|
23
|
+
rescue => boom
|
24
|
+
# won't get here in this example
|
25
|
+
circuit_breaker.mark_failure
|
26
|
+
end
|
27
|
+
else
|
28
|
+
raise "will not get here"
|
29
|
+
end
|
30
|
+
|
31
|
+
# failure
|
32
|
+
if circuit_breaker.allow_request?
|
33
|
+
begin
|
34
|
+
raise
|
35
|
+
rescue => boom
|
36
|
+
circuit_breaker.mark_failure
|
37
|
+
puts "failed slow, do fallback"
|
38
|
+
end
|
39
|
+
else
|
40
|
+
raise "will not get here"
|
41
|
+
end
|
42
|
+
|
43
|
+
# trip circuit, imagine this being same as above but in real life...
|
44
|
+
# also, we have to fail at least the request volume threshold number of times
|
45
|
+
circuit_breaker.config.request_volume_threshold.times do
|
46
|
+
circuit_breaker.mark_failure
|
47
|
+
end
|
48
|
+
|
49
|
+
# fail fast
|
50
|
+
if circuit_breaker.allow_request?
|
51
|
+
raise "will not get here"
|
52
|
+
else
|
53
|
+
puts "failed fast, do fallback"
|
54
|
+
end
|
55
|
+
|
56
|
+
now = Time.now
|
57
|
+
|
58
|
+
while Time.now - now < 3
|
59
|
+
if circuit_breaker.allow_request?
|
60
|
+
puts "doing a single attempt as we've failed fast for sleep_window_seconds"
|
61
|
+
break
|
62
|
+
else
|
63
|
+
puts "failed fast, do fallback"
|
64
|
+
end
|
65
|
+
sleep rand(0.1)
|
66
|
+
end
|
67
|
+
|
68
|
+
if circuit_breaker.allow_request?
|
69
|
+
raise "will not get here"
|
70
|
+
else
|
71
|
+
puts "request denied because single request has not been marked success yet"
|
72
|
+
end
|
73
|
+
|
74
|
+
puts "marking single request as success"
|
75
|
+
circuit_breaker.mark_success
|
76
|
+
|
77
|
+
if circuit_breaker.allow_request?
|
78
|
+
puts "circuit reset and back closed now, allowing requests"
|
79
|
+
else
|
80
|
+
raise "will not get here"
|
81
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# setting up load path
|
2
|
+
require "pathname"
|
3
|
+
root_path = Pathname(__FILE__).dirname.join("..").expand_path
|
4
|
+
lib_path = root_path.join("lib")
|
5
|
+
$:.unshift(lib_path)
|
6
|
+
|
7
|
+
# requiring stuff for this example
|
8
|
+
require "pp"
|
9
|
+
require "resilient/circuit_breaker"
|
10
|
+
|
11
|
+
config = Resilient::CircuitBreaker::Config.new({
|
12
|
+
sleep_window_seconds: 5,
|
13
|
+
request_volume_threshold: 20,
|
14
|
+
error_threshold_percentage: 10,
|
15
|
+
number_of_buckets: 10,
|
16
|
+
bucket_size_in_seconds: 1,
|
17
|
+
})
|
18
|
+
circuit_breaker = Resilient::CircuitBreaker.new(config: config)
|
19
|
+
|
20
|
+
iterations = 0
|
21
|
+
loop do
|
22
|
+
if circuit_breaker.allow_request?
|
23
|
+
begin
|
24
|
+
puts "request allowed"
|
25
|
+
raise if rand(100) < 10
|
26
|
+
puts "request succeeded"
|
27
|
+
circuit_breaker.mark_success
|
28
|
+
rescue => boom
|
29
|
+
puts "request failed"
|
30
|
+
circuit_breaker.mark_failure
|
31
|
+
end
|
32
|
+
else
|
33
|
+
puts "request denied"
|
34
|
+
end
|
35
|
+
puts "\n"
|
36
|
+
sleep 0.1
|
37
|
+
iterations += 1
|
38
|
+
|
39
|
+
if iterations % 10 == 0
|
40
|
+
p successes: circuit_breaker.metrics.successes, failures: circuit_breaker.metrics.failures, error_percentage: circuit_breaker.metrics.error_percentage, buckets: circuit_breaker.metrics.buckets.length
|
41
|
+
end
|
42
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
|
-
require "resilient/circuit_breaker/
|
2
|
-
require "resilient/circuit_breaker/
|
1
|
+
require "resilient/circuit_breaker/metrics"
|
2
|
+
require "resilient/circuit_breaker/config"
|
3
3
|
|
4
4
|
module Resilient
|
5
5
|
class CircuitBreaker
|
@@ -8,14 +8,14 @@ module Resilient
|
|
8
8
|
attr_reader :open
|
9
9
|
attr_reader :opened_or_last_checked_at_epoch
|
10
10
|
|
11
|
-
def initialize(open: false, config:
|
11
|
+
def initialize(open: false, config: Config.new, metrics: Metrics.new)
|
12
12
|
@open = open
|
13
13
|
@opened_or_last_checked_at_epoch = 0
|
14
14
|
@config = config
|
15
15
|
@metrics = if metrics
|
16
16
|
metrics
|
17
17
|
else
|
18
|
-
|
18
|
+
Metrics.new({
|
19
19
|
number_of_buckets: config.number_of_buckets,
|
20
20
|
bucket_size_in_seconds: config.bucket_size_in_seconds,
|
21
21
|
})
|
@@ -23,32 +23,74 @@ module Resilient
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def allow_request?
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
26
|
+
default_payload = {
|
27
|
+
force_open: false,
|
28
|
+
force_closed: false,
|
29
|
+
}
|
30
|
+
|
31
|
+
instrument("resilient.circuit_breaker.allow_request", default_payload) { |payload|
|
32
|
+
result = if payload[:force_open] = @config.force_open
|
33
|
+
false
|
34
|
+
else
|
35
|
+
if payload[:force_closed] = @config.force_closed
|
36
|
+
# we still want to simulate normal behavior/metrics like open, allow
|
37
|
+
# single request, etc. so it is possible to test config in
|
38
|
+
# production without impact
|
39
|
+
if payload[:open] = open?
|
40
|
+
payload[:allow_single_request] = allow_single_request?
|
41
|
+
end
|
42
|
+
|
43
|
+
true
|
44
|
+
else
|
45
|
+
if !(payload[:open] = open?)
|
46
|
+
true
|
47
|
+
else
|
48
|
+
payload[:allow_single_request] = allow_single_request?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
payload[:result] = result
|
54
|
+
}
|
30
55
|
end
|
31
56
|
|
32
57
|
def mark_success
|
33
|
-
|
58
|
+
default_payload = {
|
59
|
+
closed_the_circuit: false,
|
60
|
+
}
|
61
|
+
|
62
|
+
instrument("resilient.circuit_breaker.mark_success", default_payload) { |payload|
|
63
|
+
if @open
|
64
|
+
payload[:closed_the_circuit] = true
|
65
|
+
close_circuit
|
66
|
+
else
|
67
|
+
@metrics.mark_success
|
68
|
+
end
|
69
|
+
nil
|
70
|
+
}
|
34
71
|
end
|
35
72
|
|
36
73
|
def mark_failure
|
37
|
-
|
74
|
+
instrument("resilient.circuit_breaker.mark_failure") { |payload|
|
75
|
+
@metrics.mark_failure
|
76
|
+
nil
|
77
|
+
}
|
38
78
|
end
|
39
79
|
|
40
80
|
def reset
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
81
|
+
instrument("resilient.circuit_breaker.reset") { |payload|
|
82
|
+
@open = false
|
83
|
+
@opened_or_last_checked_at_epoch = 0
|
84
|
+
@metrics.reset
|
85
|
+
nil
|
86
|
+
}
|
45
87
|
end
|
46
88
|
|
47
89
|
private
|
48
90
|
|
49
91
|
def open_circuit
|
50
|
-
@open = true
|
51
92
|
@opened_or_last_checked_at_epoch = Time.now.to_i
|
93
|
+
@open = true
|
52
94
|
end
|
53
95
|
|
54
96
|
def close_circuit
|
@@ -71,6 +113,7 @@ module Resilient
|
|
71
113
|
return false if under_error_threshold_percentage?
|
72
114
|
|
73
115
|
open_circuit
|
116
|
+
true
|
74
117
|
end
|
75
118
|
|
76
119
|
def closed?
|
@@ -78,15 +121,18 @@ module Resilient
|
|
78
121
|
end
|
79
122
|
|
80
123
|
def allow_single_request?
|
81
|
-
try_next_request_at = @opened_or_last_checked_at_epoch + @config.sleep_window_seconds
|
82
124
|
now = Time.now.to_i
|
83
125
|
|
84
|
-
if @open && now >
|
85
|
-
@opened_or_last_checked_at_epoch = now
|
126
|
+
if @open && now > (@opened_or_last_checked_at_epoch + @config.sleep_window_seconds)
|
127
|
+
@opened_or_last_checked_at_epoch = now
|
86
128
|
true
|
87
129
|
else
|
88
130
|
false
|
89
131
|
end
|
90
132
|
end
|
133
|
+
|
134
|
+
def instrument(name, payload = {}, &block)
|
135
|
+
config.instrumenter.instrument(name, payload, &block)
|
136
|
+
end
|
91
137
|
end
|
92
138
|
end
|
@@ -1,17 +1,21 @@
|
|
1
|
+
require "resilient/instrumenters/noop"
|
2
|
+
|
1
3
|
module Resilient
|
2
4
|
class CircuitBreaker
|
3
|
-
class
|
4
|
-
attr_reader :error_threshold_percentage
|
5
|
-
attr_reader :sleep_window_seconds
|
6
|
-
attr_reader :request_volume_threshold
|
5
|
+
class Config
|
7
6
|
attr_reader :force_open
|
8
7
|
attr_reader :force_closed
|
8
|
+
attr_reader :instrumenter
|
9
|
+
attr_reader :sleep_window_seconds
|
10
|
+
attr_reader :request_volume_threshold
|
11
|
+
attr_reader :error_threshold_percentage
|
9
12
|
attr_reader :number_of_buckets
|
10
13
|
attr_reader :bucket_size_in_seconds
|
11
14
|
|
12
15
|
def initialize(options = {})
|
13
16
|
@force_open = options.fetch(:force_open, false)
|
14
17
|
@force_closed = options.fetch(:force_closed, false)
|
18
|
+
@instrumenter = options.fetch(:instrumenter, Instrumenters::Noop)
|
15
19
|
@sleep_window_seconds = options.fetch(:sleep_window_seconds, 5)
|
16
20
|
@request_volume_threshold = options.fetch(:request_volume_threshold, 20)
|
17
21
|
@error_threshold_percentage = options.fetch(:error_threshold_percentage, 50)
|
@@ -0,0 +1,164 @@
|
|
1
|
+
require "resilient/circuit_breaker/metrics/storage/memory"
|
2
|
+
|
3
|
+
module Resilient
|
4
|
+
class CircuitBreaker
|
5
|
+
class Metrics
|
6
|
+
attr_reader :number_of_buckets
|
7
|
+
attr_reader :bucket_size_in_seconds
|
8
|
+
attr_reader :buckets
|
9
|
+
attr_reader :storage
|
10
|
+
|
11
|
+
StorageSuccessKeys = [
|
12
|
+
:successes,
|
13
|
+
].freeze
|
14
|
+
|
15
|
+
StorageFailureKeys = [
|
16
|
+
:failures,
|
17
|
+
].freeze
|
18
|
+
|
19
|
+
StorageKeys = (StorageSuccessKeys + StorageFailureKeys).freeze
|
20
|
+
|
21
|
+
class Bucket
|
22
|
+
attr_reader :timestamp_start
|
23
|
+
attr_reader :timestamp_end
|
24
|
+
|
25
|
+
def initialize(timestamp_start, timestamp_end)
|
26
|
+
@timestamp_start = timestamp_start
|
27
|
+
@timestamp_end = timestamp_end
|
28
|
+
end
|
29
|
+
|
30
|
+
def prune_before(number_of_buckets, bucket_size)
|
31
|
+
@timestamp_end - (number_of_buckets * bucket_size.seconds)
|
32
|
+
end
|
33
|
+
|
34
|
+
def include?(timestamp)
|
35
|
+
timestamp >= @timestamp_start && timestamp <= @timestamp_end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class BucketRange
|
40
|
+
def self.generate(timestamp, number_of_buckets, bucket_size)
|
41
|
+
end_bucket = bucket_size.bucket(timestamp)
|
42
|
+
start_bucket = bucket_size.bucket(end_bucket.prune_before(number_of_buckets, bucket_size))
|
43
|
+
bucket_range = new(start_bucket, end_bucket)
|
44
|
+
end
|
45
|
+
|
46
|
+
attr_reader :start_bucket
|
47
|
+
attr_reader :end_bucket
|
48
|
+
|
49
|
+
def initialize(start_bucket, end_bucket)
|
50
|
+
@start_bucket = start_bucket
|
51
|
+
@end_bucket = end_bucket
|
52
|
+
end
|
53
|
+
|
54
|
+
def prune?(bucket)
|
55
|
+
bucket.timestamp_end <= @start_bucket.timestamp_end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class BucketSize
|
60
|
+
attr_reader :seconds
|
61
|
+
|
62
|
+
def initialize(seconds)
|
63
|
+
@seconds = seconds
|
64
|
+
end
|
65
|
+
|
66
|
+
def aligned_start(timestamp = Time.now.to_i)
|
67
|
+
timestamp / @seconds * @seconds
|
68
|
+
end
|
69
|
+
|
70
|
+
def aligned_end(timestamp = Time.now.to_i)
|
71
|
+
aligned_start(timestamp) + @seconds - 1
|
72
|
+
end
|
73
|
+
|
74
|
+
def bucket(timestamp = Time.now.to_i)
|
75
|
+
Bucket.new aligned_start(timestamp), aligned_end(timestamp)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def initialize(options = {})
|
80
|
+
@number_of_buckets = options.fetch(:number_of_buckets, 6)
|
81
|
+
@bucket_size = BucketSize.new(options.fetch(:bucket_size_in_seconds, 10))
|
82
|
+
@storage = options.fetch(:storage) { Storage::Memory.new }
|
83
|
+
@buckets = []
|
84
|
+
end
|
85
|
+
|
86
|
+
def mark_success
|
87
|
+
@storage.increment(current_bucket, StorageSuccessKeys)
|
88
|
+
prune_buckets
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
def mark_failure
|
93
|
+
@storage.increment(current_bucket, StorageFailureKeys)
|
94
|
+
prune_buckets
|
95
|
+
nil
|
96
|
+
end
|
97
|
+
|
98
|
+
def successes
|
99
|
+
prune_buckets
|
100
|
+
@storage.sum(@buckets, :successes)[:successes]
|
101
|
+
end
|
102
|
+
|
103
|
+
def failures
|
104
|
+
prune_buckets
|
105
|
+
@storage.sum(@buckets, :failures)[:failures]
|
106
|
+
end
|
107
|
+
|
108
|
+
def requests
|
109
|
+
prune_buckets
|
110
|
+
requests = 0
|
111
|
+
@storage.sum(@buckets, StorageKeys).each do |key, value|
|
112
|
+
requests += value
|
113
|
+
end
|
114
|
+
requests
|
115
|
+
end
|
116
|
+
|
117
|
+
def error_percentage
|
118
|
+
prune_buckets
|
119
|
+
|
120
|
+
result = @storage.sum(@buckets, StorageKeys)
|
121
|
+
successes = result[:successes]
|
122
|
+
failures = result[:failures]
|
123
|
+
|
124
|
+
requests = successes + failures
|
125
|
+
return 0 if failures == 0 || requests == 0
|
126
|
+
|
127
|
+
((failures / requests.to_f) * 100).round
|
128
|
+
end
|
129
|
+
|
130
|
+
def reset
|
131
|
+
@storage.reset(@buckets, StorageKeys)
|
132
|
+
nil
|
133
|
+
end
|
134
|
+
|
135
|
+
private
|
136
|
+
|
137
|
+
def current_bucket(timestamp = Time.now.to_i)
|
138
|
+
bucket = @buckets.detect { |bucket| bucket.include?(timestamp) }
|
139
|
+
return bucket if bucket
|
140
|
+
|
141
|
+
bucket = @bucket_size.bucket(timestamp)
|
142
|
+
@buckets.push bucket
|
143
|
+
|
144
|
+
bucket
|
145
|
+
end
|
146
|
+
|
147
|
+
def prune_buckets(timestamp = Time.now.to_i)
|
148
|
+
pruned_buckets = []
|
149
|
+
bucket_range = BucketRange.generate(timestamp, @number_of_buckets, @bucket_size)
|
150
|
+
|
151
|
+
@buckets.delete_if { |bucket|
|
152
|
+
if bucket_range.prune?(bucket)
|
153
|
+
pruned_buckets << bucket
|
154
|
+
true
|
155
|
+
end
|
156
|
+
}
|
157
|
+
|
158
|
+
if pruned_buckets.any?
|
159
|
+
@storage.prune(pruned_buckets, StorageKeys)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module Resilient
|
2
|
+
class CircuitBreaker
|
3
|
+
class Metrics
|
4
|
+
module Storage
|
5
|
+
class Memory
|
6
|
+
attr_reader :source
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@source = Hash.new { |h, k| h[k] = Hash.new(0) }
|
10
|
+
end
|
11
|
+
|
12
|
+
def increment(buckets, keys)
|
13
|
+
Array(buckets).each do |bucket|
|
14
|
+
Array(keys).each do |key|
|
15
|
+
@source[bucket][key] += 1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def get(buckets, keys)
|
21
|
+
result = Hash.new { |h, k| h[k] = Hash.new(0) }
|
22
|
+
Array(buckets).each do |bucket|
|
23
|
+
Array(keys).each do |key|
|
24
|
+
result[bucket][key] = @source[bucket][key]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
result
|
28
|
+
end
|
29
|
+
|
30
|
+
def sum(buckets, keys)
|
31
|
+
response = Hash.new(0)
|
32
|
+
Array(buckets).each do |bucket|
|
33
|
+
Array(keys).each do |key|
|
34
|
+
response[key] += @source[bucket][key]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
response
|
38
|
+
end
|
39
|
+
|
40
|
+
def reset(buckets, keys)
|
41
|
+
Array(buckets).each do |bucket|
|
42
|
+
@source.delete(bucket)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def prune(buckets, keys)
|
47
|
+
reset(buckets, keys)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Resilient
|
2
|
+
module Instrumenters
|
3
|
+
# Instrumentor that is useful for tests as it stores each of the events that
|
4
|
+
# are instrumented.
|
5
|
+
class Memory
|
6
|
+
Event = Struct.new(:name, :payload, :result)
|
7
|
+
|
8
|
+
attr_reader :events
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@events = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def instrument(name, payload = {})
|
15
|
+
# Copy the payload to guard against later modifications to it, and to
|
16
|
+
# ensure that all instrumentation code uses the payload passed to the
|
17
|
+
# block rather than the one passed to #instrument.
|
18
|
+
payload = payload.dup
|
19
|
+
|
20
|
+
result = if block_given?
|
21
|
+
yield payload
|
22
|
+
else
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
@events << Event.new(name, payload, result)
|
26
|
+
result
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module CircuitBreakerInterface
|
4
|
+
def test_responds_to_allow_request
|
5
|
+
assert_respond_to @object, :allow_request?
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_mark_success
|
9
|
+
assert_respond_to @object, :mark_success
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_mark_success_returns_nothing
|
13
|
+
assert_nil @object.mark_success
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_mark_failure
|
17
|
+
assert_respond_to @object, :mark_failure
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_mark_failure_returns_nothing
|
21
|
+
assert_nil @object.mark_failure
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_reset
|
25
|
+
assert_respond_to @object, :reset
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module ConfigInterface
|
4
|
+
def test_responds_to_force_open
|
5
|
+
assert_respond_to @object, :force_open
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_force_closed
|
9
|
+
assert_respond_to @object, :force_closed
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_instrumenter
|
13
|
+
assert_respond_to @object, :instrumenter
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_sleep_window_seconds
|
17
|
+
assert_respond_to @object, :sleep_window_seconds
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_request_volume_threshold
|
21
|
+
assert_respond_to @object, :request_volume_threshold
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_error_threshold_percentage
|
25
|
+
assert_respond_to @object, :error_threshold_percentage
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_responds_to_number_of_buckets
|
29
|
+
assert_respond_to @object, :number_of_buckets
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_responds_to_bucket_size_in_seconds
|
33
|
+
assert_respond_to @object, :bucket_size_in_seconds
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module MetricsInterface
|
4
|
+
def test_responds_to_mark_success
|
5
|
+
assert_respond_to @object, :mark_success
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_mark_success_returns_nothing
|
9
|
+
assert_nil @object.mark_success
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_mark_failure
|
13
|
+
assert_respond_to @object, :mark_failure
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_mark_failure_returns_nothing
|
17
|
+
assert_nil @object.mark_failure
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_successes
|
21
|
+
assert_respond_to @object, :successes
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_responds_to_failures
|
25
|
+
assert_respond_to @object, :failures
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_responds_to_requests
|
29
|
+
assert_respond_to @object, :requests
|
30
|
+
end
|
31
|
+
|
32
|
+
def test_responds_to_error_percentage
|
33
|
+
assert_respond_to @object, :error_percentage
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_responds_to_reset
|
37
|
+
assert_respond_to @object, :reset
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_reset_returns_nothing
|
41
|
+
assert_nil @object.reset
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module Resilient
|
2
|
+
class Test
|
3
|
+
module MetricsStorageInterface
|
4
|
+
def test_responds_to_increment
|
5
|
+
assert_respond_to @object, :increment
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_responds_to_get
|
9
|
+
assert_respond_to @object, :get
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_responds_to_sum
|
13
|
+
assert_respond_to @object, :sum
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_responds_to_prune
|
17
|
+
assert_respond_to @object, :prune
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_responds_to_reset
|
21
|
+
assert_respond_to @object, :reset
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_increment
|
25
|
+
buckets = [
|
26
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
27
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
28
|
+
]
|
29
|
+
keys = [
|
30
|
+
:successes,
|
31
|
+
:failures,
|
32
|
+
]
|
33
|
+
@object.increment(buckets, keys)
|
34
|
+
assert_equal 1, @object.source[buckets[0]][:successes]
|
35
|
+
assert_equal 1, @object.source[buckets[0]][:failures]
|
36
|
+
assert_equal 1, @object.source[buckets[1]][:successes]
|
37
|
+
assert_equal 1, @object.source[buckets[1]][:failures]
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_get_defaults
|
41
|
+
buckets = [
|
42
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
43
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
44
|
+
]
|
45
|
+
keys = [
|
46
|
+
:successes,
|
47
|
+
:failures,
|
48
|
+
]
|
49
|
+
result = @object.get(buckets, keys)
|
50
|
+
assert_equal 0, result[buckets[0]][:successes]
|
51
|
+
assert_equal 0, result[buckets[0]][:failures]
|
52
|
+
assert_equal 0, result[buckets[1]][:successes]
|
53
|
+
assert_equal 0, result[buckets[1]][:failures]
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_get_with_values
|
57
|
+
buckets = [
|
58
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
59
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
60
|
+
]
|
61
|
+
keys = [
|
62
|
+
:successes,
|
63
|
+
:failures,
|
64
|
+
]
|
65
|
+
@object.increment(buckets, keys)
|
66
|
+
@object.increment(buckets, keys)
|
67
|
+
@object.increment(buckets[0], keys)
|
68
|
+
result = @object.get(buckets, keys)
|
69
|
+
assert_equal 3, result[buckets[0]][:successes]
|
70
|
+
assert_equal 3, result[buckets[0]][:failures]
|
71
|
+
assert_equal 2, result[buckets[1]][:successes]
|
72
|
+
assert_equal 2, result[buckets[1]][:failures]
|
73
|
+
end
|
74
|
+
|
75
|
+
def test_sum
|
76
|
+
buckets = [
|
77
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
78
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
79
|
+
]
|
80
|
+
keys = [
|
81
|
+
:successes,
|
82
|
+
:failures,
|
83
|
+
]
|
84
|
+
@object.increment(buckets, keys)
|
85
|
+
@object.increment(buckets, keys)
|
86
|
+
@object.increment(buckets[0], keys)
|
87
|
+
|
88
|
+
assert_equal 5, @object.sum(buckets, [:successes])[:successes]
|
89
|
+
assert_equal 5, @object.sum(buckets, [:failures])[:failures]
|
90
|
+
assert_equal 10, @object.sum(buckets, keys).values.inject(0) { |sum, value| sum += value }
|
91
|
+
|
92
|
+
assert_equal 3, @object.sum(buckets[0], [:successes])[:successes]
|
93
|
+
assert_equal 3, @object.sum(buckets[0], [:failures])[:failures]
|
94
|
+
assert_equal 6, @object.sum(buckets[0], keys).values.inject(0) { |sum, value| sum += value }
|
95
|
+
|
96
|
+
assert_equal 2, @object.sum(buckets[1], [:successes])[:successes]
|
97
|
+
assert_equal 2, @object.sum(buckets[1], [:failures])[:failures]
|
98
|
+
assert_equal 4, @object.sum(buckets[1], keys).values.inject(0) { |sum, value| sum += value }
|
99
|
+
end
|
100
|
+
|
101
|
+
def test_reset
|
102
|
+
buckets = [
|
103
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
104
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
105
|
+
]
|
106
|
+
keys = [
|
107
|
+
:successes,
|
108
|
+
:failures,
|
109
|
+
]
|
110
|
+
@object.increment(buckets, keys)
|
111
|
+
@object.increment(buckets, keys)
|
112
|
+
@object.increment(buckets[0], keys)
|
113
|
+
@object.reset(buckets, keys)
|
114
|
+
result = @object.get(buckets, keys)
|
115
|
+
assert_equal 0, result[buckets[0]][:successes]
|
116
|
+
assert_equal 0, result[buckets[0]][:failures]
|
117
|
+
assert_equal 0, result[buckets[1]][:successes]
|
118
|
+
assert_equal 0, result[buckets[1]][:failures]
|
119
|
+
end
|
120
|
+
|
121
|
+
def test_prune
|
122
|
+
buckets = [
|
123
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(1, 5),
|
124
|
+
Resilient::CircuitBreaker::Metrics::Bucket.new(6, 10),
|
125
|
+
]
|
126
|
+
keys = [
|
127
|
+
:successes,
|
128
|
+
:failures,
|
129
|
+
]
|
130
|
+
@object.increment(buckets, keys)
|
131
|
+
@object.increment(buckets, keys)
|
132
|
+
@object.increment(buckets[0], keys)
|
133
|
+
@object.prune(buckets, keys)
|
134
|
+
result = @object.get(buckets, keys)
|
135
|
+
assert_equal 0, result[buckets[0]][:successes]
|
136
|
+
assert_equal 0, result[buckets[0]][:failures]
|
137
|
+
assert_equal 0, result[buckets[1]][:successes]
|
138
|
+
assert_equal 0, result[buckets[1]][:failures]
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
data/lib/resilient/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resilient
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Nunemaker
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-12-
|
11
|
+
date: 2015-12-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,12 +66,19 @@ files:
|
|
66
66
|
- LICENSE.txt
|
67
67
|
- README.md
|
68
68
|
- Rakefile
|
69
|
-
-
|
69
|
+
- examples/basic.rb
|
70
|
+
- examples/long_running.rb
|
70
71
|
- lib/resilient.rb
|
71
72
|
- lib/resilient/circuit_breaker.rb
|
72
|
-
- lib/resilient/circuit_breaker/
|
73
|
-
- lib/resilient/circuit_breaker/
|
74
|
-
- lib/resilient/circuit_breaker/
|
73
|
+
- lib/resilient/circuit_breaker/config.rb
|
74
|
+
- lib/resilient/circuit_breaker/metrics.rb
|
75
|
+
- lib/resilient/circuit_breaker/metrics/storage/memory.rb
|
76
|
+
- lib/resilient/instrumenters/memory.rb
|
77
|
+
- lib/resilient/instrumenters/noop.rb
|
78
|
+
- lib/resilient/test/circuit_breaker_interface.rb
|
79
|
+
- lib/resilient/test/config_interface.rb
|
80
|
+
- lib/resilient/test/metrics_interface.rb
|
81
|
+
- lib/resilient/test/metrics_storage_interface.rb
|
75
82
|
- lib/resilient/version.rb
|
76
83
|
- resilient.gemspec
|
77
84
|
- script/bootstrap
|
data/TODO.md
DELETED
@@ -1,4 +0,0 @@
|
|
1
|
-
- [ ] instrumentation
|
2
|
-
- [ ] add duration to mark success/failure; use it for tracking latency and for making circuit decisions
|
3
|
-
- [ ] add timeout to metrics separate from failures (if we add duration)?
|
4
|
-
- [ ] should force closed still instrument and all that so we can test in prod before enabling or should we allow enabling/disabling with instrumentation in a different way?
|
@@ -1,73 +0,0 @@
|
|
1
|
-
require "resilient/circuit_breaker/rolling_metrics/bucket"
|
2
|
-
|
3
|
-
module Resilient
|
4
|
-
class CircuitBreaker
|
5
|
-
class RollingMetrics
|
6
|
-
attr_reader :number_of_buckets
|
7
|
-
attr_reader :bucket_size_in_seconds
|
8
|
-
attr_reader :buckets
|
9
|
-
|
10
|
-
def initialize(options = {})
|
11
|
-
@number_of_buckets = options.fetch(:number_of_buckets, 6)
|
12
|
-
@bucket_size_in_seconds = options.fetch(:bucket_size_in_seconds, 10)
|
13
|
-
reset
|
14
|
-
end
|
15
|
-
|
16
|
-
def mark_success
|
17
|
-
timestamp = Time.now.to_i
|
18
|
-
bucket(timestamp).mark_success
|
19
|
-
prune_buckets(timestamp)
|
20
|
-
nil
|
21
|
-
end
|
22
|
-
|
23
|
-
def mark_failure
|
24
|
-
timestamp = Time.now.to_i
|
25
|
-
bucket(timestamp).mark_failure
|
26
|
-
prune_buckets(timestamp)
|
27
|
-
nil
|
28
|
-
end
|
29
|
-
|
30
|
-
def successes
|
31
|
-
prune_buckets
|
32
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.successes }
|
33
|
-
end
|
34
|
-
|
35
|
-
def failures
|
36
|
-
prune_buckets
|
37
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.failures }
|
38
|
-
end
|
39
|
-
|
40
|
-
def requests
|
41
|
-
prune_buckets
|
42
|
-
@buckets.inject(0) { |sum, bucket| sum += bucket.requests }
|
43
|
-
end
|
44
|
-
|
45
|
-
def error_percentage
|
46
|
-
return 0 if failures == 0 || requests == 0
|
47
|
-
((failures / requests.to_f) * 100).to_i
|
48
|
-
end
|
49
|
-
|
50
|
-
def reset
|
51
|
-
@buckets = []
|
52
|
-
nil
|
53
|
-
end
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def bucket(timestamp)
|
58
|
-
bucket = @buckets.detect { |bucket| bucket.include?(timestamp) }
|
59
|
-
return bucket if bucket
|
60
|
-
|
61
|
-
bucket = Bucket.new(timestamp, timestamp + @bucket_size_in_seconds - 1)
|
62
|
-
@buckets.push bucket
|
63
|
-
|
64
|
-
bucket
|
65
|
-
end
|
66
|
-
|
67
|
-
def prune_buckets(timestamp = Time.now.to_i)
|
68
|
-
cutoff = timestamp - (@number_of_buckets * @bucket_size_in_seconds)
|
69
|
-
@buckets.delete_if { |bucket| bucket.prune?(cutoff) }
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
@@ -1,37 +0,0 @@
|
|
1
|
-
module Resilient
|
2
|
-
class CircuitBreaker
|
3
|
-
class RollingMetrics
|
4
|
-
class Bucket
|
5
|
-
attr_reader :successes
|
6
|
-
attr_reader :failures
|
7
|
-
|
8
|
-
def initialize(timestamp_start, timestamp_end)
|
9
|
-
@timestamp_start = timestamp_start
|
10
|
-
@timestamp_end = timestamp_end
|
11
|
-
@successes = 0
|
12
|
-
@failures = 0
|
13
|
-
end
|
14
|
-
|
15
|
-
def mark_success
|
16
|
-
@successes += 1
|
17
|
-
end
|
18
|
-
|
19
|
-
def mark_failure
|
20
|
-
@failures += 1
|
21
|
-
end
|
22
|
-
|
23
|
-
def requests
|
24
|
-
@successes + @failures
|
25
|
-
end
|
26
|
-
|
27
|
-
def include?(timestamp)
|
28
|
-
timestamp >= @timestamp_start && timestamp <= @timestamp_end
|
29
|
-
end
|
30
|
-
|
31
|
-
def prune?(timestamp)
|
32
|
-
@timestamp_end <= timestamp
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|