gracefully 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +30 -0
- data/Gemfile +2 -0
- data/README.md +34 -1
- data/lib/gracefully.rb +22 -4
- data/lib/gracefully/all.rb +6 -0
- data/lib/gracefully/circuit_breaker.rb +88 -0
- data/lib/gracefully/command.rb +23 -0
- data/lib/gracefully/command_disabled_error.rb +6 -0
- data/lib/gracefully/consecutive_failures_based_health.rb +75 -0
- data/lib/gracefully/counter.rb +29 -0
- data/lib/gracefully/degradable.rb +36 -0
- data/lib/gracefully/{feature.rb → degradable_command.rb} +4 -3
- data/lib/gracefully/degradable_command_builder.rb +23 -0
- data/lib/gracefully/error.rb +37 -0
- data/lib/gracefully/health.rb +29 -0
- data/lib/gracefully/mutex_based_synchronized_counter.rb +30 -0
- data/lib/gracefully/retried_command.rb +25 -0
- data/lib/gracefully/short_circuited_command.rb +23 -0
- data/lib/gracefully/timed_command.rb +19 -0
- data/lib/gracefully/togglable_command.rb +20 -0
- data/lib/gracefully/try.rb +6 -34
- data/lib/gracefully/version.rb +1 -1
- data/spec/circuit_breaker_spec.rb +142 -0
- data/spec/command_spec.rb +34 -0
- data/spec/consecutive_failures_based_health_spec.rb +78 -0
- data/spec/degradable_spec.rb +49 -0
- data/spec/gracefully_spec.rb +136 -10
- data/spec/mutex_based_synchronized_counter_spec.rb +50 -0
- data/spec/retried_command_spec.rb +93 -0
- data/spec/short_circuited_command_spec.rb +136 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/timecop_helper.rb +3 -0
- data/spec/timed_command_spec.rb +73 -0
- data/spec/togglable_command_spec.rb +31 -0
- metadata +39 -5
- data/lib/gracefully/feature_builder.rb +0 -24
- data/lib/gracefully/health_meter.rb +0 -90
@@ -0,0 +1,37 @@
|
|
1
|
+
module Gracefully
|
2
|
+
# Thanks to [nested](https://github.com/skorks/nesty) for the original code
|
3
|
+
module NestedError
|
4
|
+
def initialize(message, args)
|
5
|
+
@nested = args[:nested]
|
6
|
+
super(message)
|
7
|
+
end
|
8
|
+
|
9
|
+
def set_backtrace(backtrace)
|
10
|
+
@raw_backtrace = backtrace
|
11
|
+
if nested
|
12
|
+
backtrace = include_nested_raw_backtrace_in backtrace
|
13
|
+
end
|
14
|
+
super(backtrace)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def include_nested_raw_backtrace_in(backtrace)
|
20
|
+
backtrace = backtrace - nested_raw_backtrace
|
21
|
+
backtrace += ["#{nested.backtrace.first}: #{nested.message} (#{nested.class.name})"]
|
22
|
+
backtrace + nested.backtrace[1..-1] || []
|
23
|
+
end
|
24
|
+
|
25
|
+
def nested_raw_backtrace
|
26
|
+
nested.respond_to?(:raw_backtrace) ? nested.raw_backtrace : nested.backtrace
|
27
|
+
end
|
28
|
+
|
29
|
+
def nested
|
30
|
+
@nested
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Error < StandardError
|
35
|
+
include NestedError
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Gracefully
|
2
|
+
class Health
|
3
|
+
def initialize(args)
|
4
|
+
@state = args[:state]
|
5
|
+
end
|
6
|
+
|
7
|
+
def mark_success
|
8
|
+
@state = @state.mark_success
|
9
|
+
end
|
10
|
+
|
11
|
+
def mark_failure
|
12
|
+
@state = @state.mark_failure
|
13
|
+
end
|
14
|
+
|
15
|
+
def healthy?
|
16
|
+
@state.healthy?
|
17
|
+
end
|
18
|
+
|
19
|
+
def unhealthy?
|
20
|
+
@state.unhealthy?
|
21
|
+
end
|
22
|
+
|
23
|
+
class State
|
24
|
+
def unhealthy?
|
25
|
+
!healthy?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require_relative 'counter'
|
2
|
+
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
# The counter equipped with the possibly easiest kind of synchronization.
|
7
|
+
class MutexBasedSynchronizedCounter < Counter
|
8
|
+
# @param [Counter] counter
|
9
|
+
def initialize(counter)
|
10
|
+
@counter = counter
|
11
|
+
@mutex = Mutex.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def reset!
|
15
|
+
@mutex.synchronize do
|
16
|
+
@counter.reset!
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def increment!
|
21
|
+
@mutex.synchronize do
|
22
|
+
@counter.increment!
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def count
|
27
|
+
@counter.count
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
|
3
|
+
module Gracefully
|
4
|
+
class RetriedCommand < Command
|
5
|
+
def initialize(*args, &block)
|
6
|
+
super
|
7
|
+
|
8
|
+
@retries = @options[:retries]
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(*args, &block)
|
12
|
+
num_tried = 0
|
13
|
+
begin
|
14
|
+
@callable.call *args, &block
|
15
|
+
rescue => e
|
16
|
+
num_tried += 1
|
17
|
+
if num_tried <= @retries
|
18
|
+
retry
|
19
|
+
else
|
20
|
+
raise Gracefully::Error.new(e.message, nested: e)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
require_relative 'circuit_breaker'
|
3
|
+
require_relative 'consecutive_failures_based_health'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
class ShortCircuitedCommand < Command
|
7
|
+
def initialize(*args, &block)
|
8
|
+
super
|
9
|
+
|
10
|
+
@circuit_breaker = Gracefully::CircuitBreaker.new(
|
11
|
+
try_close_after: @options[:try_close_after],
|
12
|
+
health: Gracefully::ConsecutiveFailuresBasedHealth.new(
|
13
|
+
become_unhealthy_after_consecutive_failures: @options[:allowed_failures],
|
14
|
+
counter: @options[:counter]
|
15
|
+
)
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
def call(*args, &block)
|
20
|
+
@circuit_breaker.execute { super }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
3
|
+
require_relative 'command'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
class TimedCommand < Command
|
7
|
+
def initialize(*args, &block)
|
8
|
+
super
|
9
|
+
|
10
|
+
@timeout = @options[:timeout]
|
11
|
+
end
|
12
|
+
|
13
|
+
def call(*args, &block)
|
14
|
+
Timeout.timeout(@timeout) do
|
15
|
+
@callable.call *args, &block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
require_relative 'command_disabled_error'
|
3
|
+
|
4
|
+
module Gracefully
|
5
|
+
class TogglableCommand < Command
|
6
|
+
def initialize(*args, &block)
|
7
|
+
super
|
8
|
+
|
9
|
+
@run_only_if = @options[:run_only_if]
|
10
|
+
end
|
11
|
+
|
12
|
+
def call(*args, &block)
|
13
|
+
if @run_only_if.call
|
14
|
+
@callable.call *args, &block
|
15
|
+
else
|
16
|
+
raise Gracefully::CommandDisabledError
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/lib/gracefully/try.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'error'
|
2
|
+
|
1
3
|
module Gracefully
|
2
4
|
class Try
|
3
5
|
def self.to(&block)
|
@@ -14,7 +16,9 @@ module Gracefully
|
|
14
16
|
@resolved = begin
|
15
17
|
Success.with @block.call
|
16
18
|
rescue => e
|
17
|
-
|
19
|
+
# Back-traces, which are required by Gracefully::Error, of errors are usually set by `raise`.
|
20
|
+
# We need to set them manually because we aren't relying on `raise`.
|
21
|
+
Failure.with Error.new('Nested error', nested: e).tap { |e| e.set_backtrace caller(0) }
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
@@ -59,39 +63,7 @@ module Gracefully
|
|
59
63
|
end
|
60
64
|
|
61
65
|
def get
|
62
|
-
raise @error
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
# Thanks to [nested](https://github.com/skorks/nesty) for the original code
|
67
|
-
module NestedError
|
68
|
-
def initialize(message, args)
|
69
|
-
@nested = args[:nested]
|
70
|
-
super(message)
|
71
|
-
end
|
72
|
-
|
73
|
-
def set_backtrace(backtrace)
|
74
|
-
@raw_backtrace = backtrace
|
75
|
-
if nested
|
76
|
-
backtrace = backtrace - nested_raw_backtrace
|
77
|
-
backtrace += ["#{nested.backtrace.first}: #{nested.message} (#{nested.class.name})"]
|
78
|
-
backtrace += nested.backtrace[1..-1] || []
|
79
|
-
end
|
80
|
-
super(backtrace)
|
66
|
+
raise Error.new('Tried to get the value of a failure', nested: @error)
|
81
67
|
end
|
82
|
-
|
83
|
-
private
|
84
|
-
|
85
|
-
def nested_raw_backtrace
|
86
|
-
nested.respond_to?(:raw_backtrace) ? nested.raw_backtrace : nested.backtrace
|
87
|
-
end
|
88
|
-
|
89
|
-
def nested
|
90
|
-
@nested
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
class Error < StandardError
|
95
|
-
include NestedError
|
96
68
|
end
|
97
69
|
end
|
data/lib/gracefully/version.rb
CHANGED
@@ -0,0 +1,142 @@
|
|
1
|
+
require 'timecop_helper'
|
2
|
+
|
3
|
+
require 'gracefully'
|
4
|
+
require 'gracefully/circuit_breaker'
|
5
|
+
|
6
|
+
RSpec.shared_examples 'a open circuit breaker' do
|
7
|
+
specify { expect(subject.open?).to be_truthy }
|
8
|
+
specify { expect(subject.closed?).to be_falsey }
|
9
|
+
end
|
10
|
+
|
11
|
+
RSpec.shared_examples 'a closed circuit breaker' do
|
12
|
+
specify { expect(subject.open?).to be_falsey }
|
13
|
+
specify { expect(subject.closed?).to be_truthy }
|
14
|
+
end
|
15
|
+
|
16
|
+
RSpec.shared_examples 'a circuit breaker' do
|
17
|
+
context 'when failed' do
|
18
|
+
before do
|
19
|
+
subject.mark_success
|
20
|
+
subject.mark_failure
|
21
|
+
end
|
22
|
+
|
23
|
+
it_behaves_like 'a open circuit breaker'
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'when succeeded' do
|
27
|
+
before do
|
28
|
+
subject.mark_failure
|
29
|
+
subject.mark_success
|
30
|
+
end
|
31
|
+
|
32
|
+
it_behaves_like 'a closed circuit breaker'
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'when opened' do
|
36
|
+
before do
|
37
|
+
subject.open!
|
38
|
+
end
|
39
|
+
|
40
|
+
it_behaves_like 'a open circuit breaker'
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'when closed' do
|
44
|
+
before do
|
45
|
+
subject.close!
|
46
|
+
end
|
47
|
+
|
48
|
+
it_behaves_like 'a closed circuit breaker'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
RSpec.describe Gracefully::CircuitBreaker do
|
53
|
+
context 'without try_close_period' do
|
54
|
+
subject {
|
55
|
+
described_class.new
|
56
|
+
}
|
57
|
+
|
58
|
+
it_behaves_like 'a circuit breaker'
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'with try_close_period' do
|
62
|
+
subject {
|
63
|
+
described_class.new(try_close_after: try_close_period)
|
64
|
+
}
|
65
|
+
|
66
|
+
let(:try_close_period) {
|
67
|
+
10
|
68
|
+
}
|
69
|
+
|
70
|
+
it_behaves_like 'a circuit breaker'
|
71
|
+
|
72
|
+
it 'passes the integration test' do
|
73
|
+
initial_failure_time = Time.now
|
74
|
+
|
75
|
+
Timecop.freeze(initial_failure_time) do
|
76
|
+
expect {
|
77
|
+
subject.execute do
|
78
|
+
raise 'foo'
|
79
|
+
end
|
80
|
+
}.to raise_error('foo')
|
81
|
+
|
82
|
+
expect(subject.open?).to be_truthy
|
83
|
+
expect(subject.closed?).to be_falsey
|
84
|
+
expect(subject.opened_date).not_to be_nil
|
85
|
+
end
|
86
|
+
|
87
|
+
Timecop.freeze(initial_failure_time + 10) do
|
88
|
+
expect {
|
89
|
+
subject.execute do
|
90
|
+
end
|
91
|
+
}.to raise_error(Gracefully::CircuitBreaker::CurrentlyOpenError)
|
92
|
+
|
93
|
+
expect(subject.open?).to be_truthy
|
94
|
+
expect(subject.closed?).to be_falsey
|
95
|
+
expect(subject.opened_date).not_to be_nil
|
96
|
+
end
|
97
|
+
|
98
|
+
Timecop.freeze(initial_failure_time + 11) do
|
99
|
+
expect(
|
100
|
+
subject.execute do
|
101
|
+
'baz'
|
102
|
+
end
|
103
|
+
).to eq('baz')
|
104
|
+
|
105
|
+
expect(subject.open?).to be_falsey
|
106
|
+
expect(subject.closed?).to be_truthy
|
107
|
+
expect(subject.try_close_period_passed?).to be_falsey
|
108
|
+
expect(subject.opened_date).to be_nil
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
context 'when try-close period passed after its opened' do
|
113
|
+
before do
|
114
|
+
Timecop.freeze(failed_date) do
|
115
|
+
subject.mark_failure
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
let(:failed_date) {
|
120
|
+
Time.now
|
121
|
+
}
|
122
|
+
|
123
|
+
let(:after_date) {
|
124
|
+
failed_date + 11
|
125
|
+
}
|
126
|
+
|
127
|
+
specify { expect(subject.open?).to be_truthy }
|
128
|
+
specify { expect(subject.closed?).to be_falsey }
|
129
|
+
specify { expect(subject.opened_date).to eq(failed_date) }
|
130
|
+
specify {
|
131
|
+
Timecop.freeze(after_date) do
|
132
|
+
expect(subject.try_close_period_passed?).to be_truthy
|
133
|
+
end
|
134
|
+
}
|
135
|
+
specify {
|
136
|
+
Timecop.freeze(failed_date) do
|
137
|
+
expect(subject.try_close_period_passed?).to be_falsey
|
138
|
+
end
|
139
|
+
}
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'gracefully'
|
2
|
+
require 'gracefully/command'
|
3
|
+
|
4
|
+
RSpec.shared_examples 'a command' do
|
5
|
+
let(:input1) {
|
6
|
+
'input1'
|
7
|
+
}
|
8
|
+
|
9
|
+
context 'when the block given to the command succeeds without any error' do
|
10
|
+
let(:usually) {
|
11
|
+
-> arg1 { 'usually:arg1:' + arg1 }
|
12
|
+
}
|
13
|
+
|
14
|
+
it { is_expected.to eq('usually:arg1:input1') }
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'when the usually block fails with an error' do
|
18
|
+
let(:usually) {
|
19
|
+
-> arg1 { raise 'simulated error' }
|
20
|
+
}
|
21
|
+
|
22
|
+
specify { expect { subject }.to raise_error('simulated error') }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
RSpec.describe Gracefully::Command do
|
27
|
+
describe "feature call result" do
|
28
|
+
subject {
|
29
|
+
described_class.new(&usually).call input1
|
30
|
+
}
|
31
|
+
|
32
|
+
it_behaves_like 'a command'
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'gracefully/consecutive_failures_based_health'
|
2
|
+
|
3
|
+
RSpec.shared_examples 'healthy' do
|
4
|
+
it { is_expected.to be_healthy }
|
5
|
+
it { is_expected.not_to be_unhealthy }
|
6
|
+
end
|
7
|
+
|
8
|
+
RSpec.shared_examples 'unhealthy' do
|
9
|
+
it { is_expected.to be_unhealthy }
|
10
|
+
it { is_expected.not_to be_healthy }
|
11
|
+
end
|
12
|
+
|
13
|
+
RSpec.describe Gracefully::ConsecutiveFailuresBasedHealth do
|
14
|
+
subject {
|
15
|
+
described_class.new(
|
16
|
+
become_unhealthy_after_consecutive_failures: threshold,
|
17
|
+
counter: -> { Gracefully::InMemoryCounter.new }
|
18
|
+
)
|
19
|
+
}
|
20
|
+
|
21
|
+
let(:threshold) {
|
22
|
+
1
|
23
|
+
}
|
24
|
+
|
25
|
+
def less_than_or_equal_to(t)
|
26
|
+
t
|
27
|
+
end
|
28
|
+
|
29
|
+
def more_than(t)
|
30
|
+
t + 1
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'initially' do
|
34
|
+
it_behaves_like 'healthy'
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'after failures less than or equal to the threshold' do
|
38
|
+
before do
|
39
|
+
less_than_or_equal_to(threshold).times { subject.mark_failure }
|
40
|
+
end
|
41
|
+
|
42
|
+
it_behaves_like 'healthy'
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'after failures more than the threshold' do
|
46
|
+
before do
|
47
|
+
more_than(threshold).times { subject.mark_failure }
|
48
|
+
end
|
49
|
+
|
50
|
+
it_behaves_like 'unhealthy'
|
51
|
+
end
|
52
|
+
|
53
|
+
context 'after failures more than threshold and a success' do
|
54
|
+
before do
|
55
|
+
more_than(threshold).times { subject.mark_failure }
|
56
|
+
subject.mark_success
|
57
|
+
end
|
58
|
+
|
59
|
+
it_behaves_like 'healthy'
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'after failurse more than thoreshold + 1 and a success' do
|
63
|
+
before do
|
64
|
+
more_than(threshold + 1).times { subject.mark_failure }
|
65
|
+
subject.mark_success
|
66
|
+
end
|
67
|
+
|
68
|
+
it_behaves_like 'healthy'
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'after consecutive successes' do
|
72
|
+
before do
|
73
|
+
2.times { subject.mark_success }
|
74
|
+
end
|
75
|
+
|
76
|
+
it_behaves_like 'healthy'
|
77
|
+
end
|
78
|
+
end
|