gracefully 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +30 -0
- data/Gemfile +2 -0
- data/README.md +34 -1
- data/lib/gracefully.rb +22 -4
- data/lib/gracefully/all.rb +6 -0
- data/lib/gracefully/circuit_breaker.rb +88 -0
- data/lib/gracefully/command.rb +23 -0
- data/lib/gracefully/command_disabled_error.rb +6 -0
- data/lib/gracefully/consecutive_failures_based_health.rb +75 -0
- data/lib/gracefully/counter.rb +29 -0
- data/lib/gracefully/degradable.rb +36 -0
- data/lib/gracefully/{feature.rb → degradable_command.rb} +4 -3
- data/lib/gracefully/degradable_command_builder.rb +23 -0
- data/lib/gracefully/error.rb +37 -0
- data/lib/gracefully/health.rb +29 -0
- data/lib/gracefully/mutex_based_synchronized_counter.rb +30 -0
- data/lib/gracefully/retried_command.rb +25 -0
- data/lib/gracefully/short_circuited_command.rb +23 -0
- data/lib/gracefully/timed_command.rb +19 -0
- data/lib/gracefully/togglable_command.rb +20 -0
- data/lib/gracefully/try.rb +6 -34
- data/lib/gracefully/version.rb +1 -1
- data/spec/circuit_breaker_spec.rb +142 -0
- data/spec/command_spec.rb +34 -0
- data/spec/consecutive_failures_based_health_spec.rb +78 -0
- data/spec/degradable_spec.rb +49 -0
- data/spec/gracefully_spec.rb +136 -10
- data/spec/mutex_based_synchronized_counter_spec.rb +50 -0
- data/spec/retried_command_spec.rb +93 -0
- data/spec/short_circuited_command_spec.rb +136 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/timecop_helper.rb +3 -0
- data/spec/timed_command_spec.rb +73 -0
- data/spec/togglable_command_spec.rb +31 -0
- metadata +39 -5
- data/lib/gracefully/feature_builder.rb +0 -24
- data/lib/gracefully/health_meter.rb +0 -90
@@ -0,0 +1,37 @@
|
|
1
|
+
module Gracefully
|
2
|
+
# Thanks to [nested](https://github.com/skorks/nesty) for the original code
|
3
|
+
module NestedError
|
4
|
+
def initialize(message, args)
|
5
|
+
@nested = args[:nested]
|
6
|
+
super(message)
|
7
|
+
end
|
8
|
+
|
9
|
+
def set_backtrace(backtrace)
|
10
|
+
@raw_backtrace = backtrace
|
11
|
+
if nested
|
12
|
+
backtrace = include_nested_raw_backtrace_in backtrace
|
13
|
+
end
|
14
|
+
super(backtrace)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def include_nested_raw_backtrace_in(backtrace)
|
20
|
+
backtrace = backtrace - nested_raw_backtrace
|
21
|
+
backtrace += ["#{nested.backtrace.first}: #{nested.message} (#{nested.class.name})"]
|
22
|
+
backtrace + nested.backtrace[1..-1] || []
|
23
|
+
end
|
24
|
+
|
25
|
+
def nested_raw_backtrace
|
26
|
+
nested.respond_to?(:raw_backtrace) ? nested.raw_backtrace : nested.backtrace
|
27
|
+
end
|
28
|
+
|
29
|
+
def nested
|
30
|
+
@nested
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Error < StandardError
|
35
|
+
include NestedError
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Gracefully
|
2
|
+
class Health
|
3
|
+
def initialize(args)
|
4
|
+
@state = args[:state]
|
5
|
+
end
|
6
|
+
|
7
|
+
def mark_success
|
8
|
+
@state = @state.mark_success
|
9
|
+
end
|
10
|
+
|
11
|
+
def mark_failure
|
12
|
+
@state = @state.mark_failure
|
13
|
+
end
|
14
|
+
|
15
|
+
def healthy?
|
16
|
+
@state.healthy?
|
17
|
+
end
|
18
|
+
|
19
|
+
def unhealthy?
|
20
|
+
@state.unhealthy?
|
21
|
+
end
|
22
|
+
|
23
|
+
class State
|
24
|
+
def unhealthy?
|
25
|
+
!healthy?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require_relative 'counter'
|
2
|
+
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
# The counter equipped with the possibly easiest kind of synchronization.
|
7
|
+
class MutexBasedSynchronizedCounter < Counter
|
8
|
+
# @param [Counter] counter
|
9
|
+
def initialize(counter)
|
10
|
+
@counter = counter
|
11
|
+
@mutex = Mutex.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def reset!
|
15
|
+
@mutex.synchronize do
|
16
|
+
@counter.reset!
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def increment!
|
21
|
+
@mutex.synchronize do
|
22
|
+
@counter.increment!
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def count
|
27
|
+
@counter.count
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
|
3
|
+
module Gracefully
|
4
|
+
class RetriedCommand < Command
|
5
|
+
def initialize(*args, &block)
|
6
|
+
super
|
7
|
+
|
8
|
+
@retries = @options[:retries]
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(*args, &block)
|
12
|
+
num_tried = 0
|
13
|
+
begin
|
14
|
+
@callable.call *args, &block
|
15
|
+
rescue => e
|
16
|
+
num_tried += 1
|
17
|
+
if num_tried <= @retries
|
18
|
+
retry
|
19
|
+
else
|
20
|
+
raise Gracefully::Error.new(e.message, nested: e)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
require_relative 'circuit_breaker'
|
3
|
+
require_relative 'consecutive_failures_based_health'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
class ShortCircuitedCommand < Command
|
7
|
+
def initialize(*args, &block)
|
8
|
+
super
|
9
|
+
|
10
|
+
@circuit_breaker = Gracefully::CircuitBreaker.new(
|
11
|
+
try_close_after: @options[:try_close_after],
|
12
|
+
health: Gracefully::ConsecutiveFailuresBasedHealth.new(
|
13
|
+
become_unhealthy_after_consecutive_failures: @options[:allowed_failures],
|
14
|
+
counter: @options[:counter]
|
15
|
+
)
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
def call(*args, &block)
|
20
|
+
@circuit_breaker.execute { super }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
3
|
+
require_relative 'command'
|
4
|
+
|
5
|
+
module Gracefully
|
6
|
+
class TimedCommand < Command
|
7
|
+
def initialize(*args, &block)
|
8
|
+
super
|
9
|
+
|
10
|
+
@timeout = @options[:timeout]
|
11
|
+
end
|
12
|
+
|
13
|
+
def call(*args, &block)
|
14
|
+
Timeout.timeout(@timeout) do
|
15
|
+
@callable.call *args, &block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative 'command'
|
2
|
+
require_relative 'command_disabled_error'
|
3
|
+
|
4
|
+
module Gracefully
|
5
|
+
class TogglableCommand < Command
|
6
|
+
def initialize(*args, &block)
|
7
|
+
super
|
8
|
+
|
9
|
+
@run_only_if = @options[:run_only_if]
|
10
|
+
end
|
11
|
+
|
12
|
+
def call(*args, &block)
|
13
|
+
if @run_only_if.call
|
14
|
+
@callable.call *args, &block
|
15
|
+
else
|
16
|
+
raise Gracefully::CommandDisabledError
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/lib/gracefully/try.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'error'
|
2
|
+
|
1
3
|
module Gracefully
|
2
4
|
class Try
|
3
5
|
def self.to(&block)
|
@@ -14,7 +16,9 @@ module Gracefully
|
|
14
16
|
@resolved = begin
|
15
17
|
Success.with @block.call
|
16
18
|
rescue => e
|
17
|
-
|
19
|
+
# Back-traces, which are required by Gracefully::Error, of errors are usually set by `raise`.
|
20
|
+
# We need to set them manually because we aren't relying on `raise`.
|
21
|
+
Failure.with Error.new('Nested error', nested: e).tap { |e| e.set_backtrace caller(0) }
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
@@ -59,39 +63,7 @@ module Gracefully
|
|
59
63
|
end
|
60
64
|
|
61
65
|
def get
|
62
|
-
raise @error
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
# Thanks to [nested](https://github.com/skorks/nesty) for the original code
|
67
|
-
module NestedError
|
68
|
-
def initialize(message, args)
|
69
|
-
@nested = args[:nested]
|
70
|
-
super(message)
|
71
|
-
end
|
72
|
-
|
73
|
-
def set_backtrace(backtrace)
|
74
|
-
@raw_backtrace = backtrace
|
75
|
-
if nested
|
76
|
-
backtrace = backtrace - nested_raw_backtrace
|
77
|
-
backtrace += ["#{nested.backtrace.first}: #{nested.message} (#{nested.class.name})"]
|
78
|
-
backtrace += nested.backtrace[1..-1] || []
|
79
|
-
end
|
80
|
-
super(backtrace)
|
66
|
+
raise Error.new('Tried to get the value of a failure', nested: @error)
|
81
67
|
end
|
82
|
-
|
83
|
-
private
|
84
|
-
|
85
|
-
def nested_raw_backtrace
|
86
|
-
nested.respond_to?(:raw_backtrace) ? nested.raw_backtrace : nested.backtrace
|
87
|
-
end
|
88
|
-
|
89
|
-
def nested
|
90
|
-
@nested
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
class Error < StandardError
|
95
|
-
include NestedError
|
96
68
|
end
|
97
69
|
end
|
data/lib/gracefully/version.rb
CHANGED
@@ -0,0 +1,142 @@
|
|
1
|
+
require 'timecop_helper'
|
2
|
+
|
3
|
+
require 'gracefully'
|
4
|
+
require 'gracefully/circuit_breaker'
|
5
|
+
|
6
|
+
RSpec.shared_examples 'a open circuit breaker' do
|
7
|
+
specify { expect(subject.open?).to be_truthy }
|
8
|
+
specify { expect(subject.closed?).to be_falsey }
|
9
|
+
end
|
10
|
+
|
11
|
+
RSpec.shared_examples 'a closed circuit breaker' do
|
12
|
+
specify { expect(subject.open?).to be_falsey }
|
13
|
+
specify { expect(subject.closed?).to be_truthy }
|
14
|
+
end
|
15
|
+
|
16
|
+
RSpec.shared_examples 'a circuit breaker' do
|
17
|
+
context 'when failed' do
|
18
|
+
before do
|
19
|
+
subject.mark_success
|
20
|
+
subject.mark_failure
|
21
|
+
end
|
22
|
+
|
23
|
+
it_behaves_like 'a open circuit breaker'
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'when succeeded' do
|
27
|
+
before do
|
28
|
+
subject.mark_failure
|
29
|
+
subject.mark_success
|
30
|
+
end
|
31
|
+
|
32
|
+
it_behaves_like 'a closed circuit breaker'
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'when opened' do
|
36
|
+
before do
|
37
|
+
subject.open!
|
38
|
+
end
|
39
|
+
|
40
|
+
it_behaves_like 'a open circuit breaker'
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'when closed' do
|
44
|
+
before do
|
45
|
+
subject.close!
|
46
|
+
end
|
47
|
+
|
48
|
+
it_behaves_like 'a closed circuit breaker'
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
RSpec.describe Gracefully::CircuitBreaker do
|
53
|
+
context 'without try_close_period' do
|
54
|
+
subject {
|
55
|
+
described_class.new
|
56
|
+
}
|
57
|
+
|
58
|
+
it_behaves_like 'a circuit breaker'
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'with try_close_period' do
|
62
|
+
subject {
|
63
|
+
described_class.new(try_close_after: try_close_period)
|
64
|
+
}
|
65
|
+
|
66
|
+
let(:try_close_period) {
|
67
|
+
10
|
68
|
+
}
|
69
|
+
|
70
|
+
it_behaves_like 'a circuit breaker'
|
71
|
+
|
72
|
+
it 'passes the integration test' do
|
73
|
+
initial_failure_time = Time.now
|
74
|
+
|
75
|
+
Timecop.freeze(initial_failure_time) do
|
76
|
+
expect {
|
77
|
+
subject.execute do
|
78
|
+
raise 'foo'
|
79
|
+
end
|
80
|
+
}.to raise_error('foo')
|
81
|
+
|
82
|
+
expect(subject.open?).to be_truthy
|
83
|
+
expect(subject.closed?).to be_falsey
|
84
|
+
expect(subject.opened_date).not_to be_nil
|
85
|
+
end
|
86
|
+
|
87
|
+
Timecop.freeze(initial_failure_time + 10) do
|
88
|
+
expect {
|
89
|
+
subject.execute do
|
90
|
+
end
|
91
|
+
}.to raise_error(Gracefully::CircuitBreaker::CurrentlyOpenError)
|
92
|
+
|
93
|
+
expect(subject.open?).to be_truthy
|
94
|
+
expect(subject.closed?).to be_falsey
|
95
|
+
expect(subject.opened_date).not_to be_nil
|
96
|
+
end
|
97
|
+
|
98
|
+
Timecop.freeze(initial_failure_time + 11) do
|
99
|
+
expect(
|
100
|
+
subject.execute do
|
101
|
+
'baz'
|
102
|
+
end
|
103
|
+
).to eq('baz')
|
104
|
+
|
105
|
+
expect(subject.open?).to be_falsey
|
106
|
+
expect(subject.closed?).to be_truthy
|
107
|
+
expect(subject.try_close_period_passed?).to be_falsey
|
108
|
+
expect(subject.opened_date).to be_nil
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
context 'when try-close period passed after its opened' do
|
113
|
+
before do
|
114
|
+
Timecop.freeze(failed_date) do
|
115
|
+
subject.mark_failure
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
let(:failed_date) {
|
120
|
+
Time.now
|
121
|
+
}
|
122
|
+
|
123
|
+
let(:after_date) {
|
124
|
+
failed_date + 11
|
125
|
+
}
|
126
|
+
|
127
|
+
specify { expect(subject.open?).to be_truthy }
|
128
|
+
specify { expect(subject.closed?).to be_falsey }
|
129
|
+
specify { expect(subject.opened_date).to eq(failed_date) }
|
130
|
+
specify {
|
131
|
+
Timecop.freeze(after_date) do
|
132
|
+
expect(subject.try_close_period_passed?).to be_truthy
|
133
|
+
end
|
134
|
+
}
|
135
|
+
specify {
|
136
|
+
Timecop.freeze(failed_date) do
|
137
|
+
expect(subject.try_close_period_passed?).to be_falsey
|
138
|
+
end
|
139
|
+
}
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'gracefully'
|
2
|
+
require 'gracefully/command'
|
3
|
+
|
4
|
+
RSpec.shared_examples 'a command' do
|
5
|
+
let(:input1) {
|
6
|
+
'input1'
|
7
|
+
}
|
8
|
+
|
9
|
+
context 'when the block given to the command succeeds without any error' do
|
10
|
+
let(:usually) {
|
11
|
+
-> arg1 { 'usually:arg1:' + arg1 }
|
12
|
+
}
|
13
|
+
|
14
|
+
it { is_expected.to eq('usually:arg1:input1') }
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'when the usually block fails with an error' do
|
18
|
+
let(:usually) {
|
19
|
+
-> arg1 { raise 'simulated error' }
|
20
|
+
}
|
21
|
+
|
22
|
+
specify { expect { subject }.to raise_error('simulated error') }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
RSpec.describe Gracefully::Command do
|
27
|
+
describe "feature call result" do
|
28
|
+
subject {
|
29
|
+
described_class.new(&usually).call input1
|
30
|
+
}
|
31
|
+
|
32
|
+
it_behaves_like 'a command'
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'gracefully/consecutive_failures_based_health'
|
2
|
+
|
3
|
+
RSpec.shared_examples 'healthy' do
|
4
|
+
it { is_expected.to be_healthy }
|
5
|
+
it { is_expected.not_to be_unhealthy }
|
6
|
+
end
|
7
|
+
|
8
|
+
RSpec.shared_examples 'unhealthy' do
|
9
|
+
it { is_expected.to be_unhealthy }
|
10
|
+
it { is_expected.not_to be_healthy }
|
11
|
+
end
|
12
|
+
|
13
|
+
RSpec.describe Gracefully::ConsecutiveFailuresBasedHealth do
|
14
|
+
subject {
|
15
|
+
described_class.new(
|
16
|
+
become_unhealthy_after_consecutive_failures: threshold,
|
17
|
+
counter: -> { Gracefully::InMemoryCounter.new }
|
18
|
+
)
|
19
|
+
}
|
20
|
+
|
21
|
+
let(:threshold) {
|
22
|
+
1
|
23
|
+
}
|
24
|
+
|
25
|
+
def less_than_or_equal_to(t)
|
26
|
+
t
|
27
|
+
end
|
28
|
+
|
29
|
+
def more_than(t)
|
30
|
+
t + 1
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'initially' do
|
34
|
+
it_behaves_like 'healthy'
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'after failures less than or equal to the threshold' do
|
38
|
+
before do
|
39
|
+
less_than_or_equal_to(threshold).times { subject.mark_failure }
|
40
|
+
end
|
41
|
+
|
42
|
+
it_behaves_like 'healthy'
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'after failures more than the threshold' do
|
46
|
+
before do
|
47
|
+
more_than(threshold).times { subject.mark_failure }
|
48
|
+
end
|
49
|
+
|
50
|
+
it_behaves_like 'unhealthy'
|
51
|
+
end
|
52
|
+
|
53
|
+
context 'after failures more than threshold and a success' do
|
54
|
+
before do
|
55
|
+
more_than(threshold).times { subject.mark_failure }
|
56
|
+
subject.mark_success
|
57
|
+
end
|
58
|
+
|
59
|
+
it_behaves_like 'healthy'
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'after failurse more than thoreshold + 1 and a success' do
|
63
|
+
before do
|
64
|
+
more_than(threshold + 1).times { subject.mark_failure }
|
65
|
+
subject.mark_success
|
66
|
+
end
|
67
|
+
|
68
|
+
it_behaves_like 'healthy'
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'after consecutive successes' do
|
72
|
+
before do
|
73
|
+
2.times { subject.mark_success }
|
74
|
+
end
|
75
|
+
|
76
|
+
it_behaves_like 'healthy'
|
77
|
+
end
|
78
|
+
end
|