stoplight 4.1.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +289 -350
- data/lib/stoplight/admin/actions/action.rb +24 -0
- data/lib/stoplight/admin/actions/lock.rb +23 -0
- data/lib/stoplight/admin/actions/lock_all_green.rb +18 -0
- data/lib/stoplight/admin/actions/lock_green.rb +23 -0
- data/lib/stoplight/admin/actions/lock_red.rb +23 -0
- data/lib/stoplight/admin/actions/stats.rb +27 -0
- data/lib/stoplight/admin/actions/unlock.rb +23 -0
- data/lib/stoplight/admin/dependencies.rb +50 -0
- data/lib/stoplight/admin/helpers.rb +27 -0
- data/lib/stoplight/admin/lights_repository/light.rb +155 -0
- data/lib/stoplight/admin/lights_repository.rb +74 -0
- data/lib/stoplight/admin/lights_stats.rb +77 -0
- data/lib/stoplight/admin/views/_card.erb +120 -0
- data/lib/stoplight/admin/views/index.erb +36 -0
- data/lib/stoplight/admin/views/layout.erb +66 -0
- data/lib/stoplight/admin.rb +68 -0
- data/lib/stoplight/color.rb +3 -3
- data/lib/stoplight/config/config_provider.rb +62 -0
- data/lib/stoplight/config/library_default_config.rb +29 -0
- data/lib/stoplight/config/user_default_config.rb +83 -0
- data/lib/stoplight/data_store/base.rb +59 -33
- data/lib/stoplight/data_store/fail_safe.rb +105 -0
- data/lib/stoplight/data_store/memory.rb +257 -50
- data/lib/stoplight/data_store/redis/get_metadata.lua +38 -0
- data/lib/stoplight/data_store/redis/lua.rb +23 -0
- data/lib/stoplight/data_store/redis/record_failure.lua +36 -0
- data/lib/stoplight/data_store/redis/record_success.lua +35 -0
- data/lib/stoplight/data_store/redis/transition_to_green.lua +10 -0
- data/lib/stoplight/data_store/redis/transition_to_red.lua +10 -0
- data/lib/stoplight/data_store/redis/transition_to_yellow.lua +9 -0
- data/lib/stoplight/data_store/redis.rb +345 -106
- data/lib/stoplight/default.rb +11 -9
- data/lib/stoplight/error.rb +1 -13
- data/lib/stoplight/failure.rb +14 -13
- data/lib/stoplight/light/config.rb +118 -0
- data/lib/stoplight/light/configuration_builder_interface.rb +128 -0
- data/lib/stoplight/light/green_run_strategy.rb +53 -0
- data/lib/stoplight/light/red_run_strategy.rb +26 -0
- data/lib/stoplight/light/run_strategy.rb +30 -0
- data/lib/stoplight/light/yellow_run_strategy.rb +78 -0
- data/lib/stoplight/light.rb +164 -84
- data/lib/stoplight/metadata.rb +71 -0
- data/lib/stoplight/notifier/base.rb +14 -7
- data/lib/stoplight/notifier/fail_safe.rb +67 -0
- data/lib/stoplight/notifier/generic.rb +54 -5
- data/lib/stoplight/rspec/generic_notifier.rb +11 -12
- data/lib/stoplight/rspec.rb +1 -1
- data/lib/stoplight/state.rb +3 -3
- data/lib/stoplight/traffic_control/base.rb +35 -0
- data/lib/stoplight/traffic_control/consecutive_failures.rb +43 -0
- data/lib/stoplight/traffic_recovery/base.rb +51 -0
- data/lib/stoplight/traffic_recovery/single_success.rb +35 -0
- data/lib/stoplight/version.rb +1 -1
- data/lib/stoplight.rb +111 -51
- metadata +49 -98
- data/lib/stoplight/builder.rb +0 -70
- data/lib/stoplight/circuit_breaker.rb +0 -102
- data/lib/stoplight/configurable.rb +0 -95
- data/lib/stoplight/configuration.rb +0 -126
- data/lib/stoplight/light/deprecated.rb +0 -44
- data/lib/stoplight/light/lockable.rb +0 -45
- data/lib/stoplight/light/runnable.rb +0 -127
- data/lib/stoplight/notifier.rb +0 -6
- data/spec/spec_helper.rb +0 -22
- data/spec/stoplight/builder_spec.rb +0 -165
- data/spec/stoplight/circuit_breaker_spec.rb +0 -43
- data/spec/stoplight/color_spec.rb +0 -39
- data/spec/stoplight/configurable_spec.rb +0 -25
- data/spec/stoplight/data_store/base_spec.rb +0 -71
- data/spec/stoplight/data_store/memory_spec.rb +0 -22
- data/spec/stoplight/data_store/redis_spec.rb +0 -45
- data/spec/stoplight/data_store_spec.rb +0 -9
- data/spec/stoplight/default_spec.rb +0 -80
- data/spec/stoplight/error_spec.rb +0 -39
- data/spec/stoplight/failure_spec.rb +0 -108
- data/spec/stoplight/light/lockable_spec.rb +0 -93
- data/spec/stoplight/light/runnable_spec.rb +0 -38
- data/spec/stoplight/light_spec.rb +0 -156
- data/spec/stoplight/notifier/base_spec.rb +0 -18
- data/spec/stoplight/notifier/generic_spec.rb +0 -50
- data/spec/stoplight/notifier/io_spec.rb +0 -41
- data/spec/stoplight/notifier/logger_spec.rb +0 -75
- data/spec/stoplight/notifier_spec.rb +0 -9
- data/spec/stoplight/state_spec.rb +0 -39
- data/spec/stoplight/version_spec.rb +0 -9
- data/spec/stoplight_spec.rb +0 -32
- data/spec/support/configurable.rb +0 -69
- data/spec/support/data_store/base/clear_failures.rb +0 -18
- data/spec/support/data_store/base/clear_state.rb +0 -20
- data/spec/support/data_store/base/get_all.rb +0 -44
- data/spec/support/data_store/base/get_failures.rb +0 -30
- data/spec/support/data_store/base/get_state.rb +0 -7
- data/spec/support/data_store/base/names.rb +0 -29
- data/spec/support/data_store/base/record_failures.rb +0 -70
- data/spec/support/data_store/base/set_state.rb +0 -15
- data/spec/support/data_store/base/with_notification_lock.rb +0 -27
- data/spec/support/data_store/base.rb +0 -21
- data/spec/support/database_cleaner.rb +0 -26
- data/spec/support/exception_helpers.rb +0 -9
- data/spec/support/light/runnable/color.rb +0 -79
- data/spec/support/light/runnable/run.rb +0 -247
- data/spec/support/light/runnable/state.rb +0 -31
- data/spec/support/light/runnable.rb +0 -5
@@ -0,0 +1,118 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Stoplight
|
4
|
+
class Light
|
5
|
+
# A +Stoplight::Light+ configuration object.
|
6
|
+
# @api private
|
7
|
+
class Config
|
8
|
+
# @!attribute [r] name
|
9
|
+
# @return [String]
|
10
|
+
attr_reader :name
|
11
|
+
|
12
|
+
# @!attribute [r] cool_off_time
|
13
|
+
# @return [Numeric]
|
14
|
+
attr_reader :cool_off_time
|
15
|
+
|
16
|
+
# @!attribute [r] data_store
|
17
|
+
# @return [Stoplight::DataStore::Base]
|
18
|
+
attr_reader :data_store
|
19
|
+
|
20
|
+
# @!attribute [r] error_notifier
|
21
|
+
# @return [StandardError => void]
|
22
|
+
attr_reader :error_notifier
|
23
|
+
|
24
|
+
# @!attribute [r] notifiers
|
25
|
+
# @return [Array<Stoplight::Notifier::Base>]
|
26
|
+
attr_reader :notifiers
|
27
|
+
|
28
|
+
# @!attribute [r] threshold
|
29
|
+
# @return [Numeric]
|
30
|
+
attr_reader :threshold
|
31
|
+
|
32
|
+
# @!attribute [r] window_size
|
33
|
+
# @return [Numeric]
|
34
|
+
attr_reader :window_size
|
35
|
+
|
36
|
+
# @!attribute [r] tracked_errors
|
37
|
+
# @return [Array<StandardError>]
|
38
|
+
attr_reader :tracked_errors
|
39
|
+
|
40
|
+
# @!attribute [r] skipped_errors
|
41
|
+
# @return [Array<Exception>]
|
42
|
+
attr_reader :skipped_errors
|
43
|
+
|
44
|
+
# @!attribute [r] traffic_control
|
45
|
+
# @return [Stoplight::TrafficControl::Base]
|
46
|
+
attr_reader :traffic_control
|
47
|
+
|
48
|
+
# @!attribute [r] traffic_recovery
|
49
|
+
# @return [Stoplight::TrafficRecovery::Base]
|
50
|
+
attr_reader :traffic_recovery
|
51
|
+
|
52
|
+
# @param name [String]
|
53
|
+
# @param cool_off_time [Numeric]
|
54
|
+
# @param data_store [Stoplight::DataStore::Base]
|
55
|
+
# @param error_notifier [Proc]
|
56
|
+
# @param notifiers [Array<Stoplight::Notifier::Base>]
|
57
|
+
# @param threshold [Numeric]
|
58
|
+
# @param window_size [Numeric]
|
59
|
+
# @param tracked_errors [Array<StandardError>]
|
60
|
+
# @param skipped_errors [Array<Exception>]
|
61
|
+
# @param traffic_control [Stoplight::TrafficControl::Base]
|
62
|
+
# @param traffic_recovery [Stoplight::TrafficRecovery::Base]
|
63
|
+
def initialize(name:, cool_off_time:, data_store:, error_notifier:, notifiers:, threshold:, window_size:,
|
64
|
+
tracked_errors:, skipped_errors:, traffic_control:, traffic_recovery:)
|
65
|
+
@name = name
|
66
|
+
@cool_off_time = cool_off_time.to_i
|
67
|
+
@data_store = DataStore::FailSafe.wrap(data_store)
|
68
|
+
@error_notifier = error_notifier
|
69
|
+
@notifiers = notifiers.map { |notifier| Notifier::FailSafe.wrap(notifier) }
|
70
|
+
@threshold = threshold
|
71
|
+
@window_size = window_size
|
72
|
+
@tracked_errors = Array(tracked_errors)
|
73
|
+
@skipped_errors = Array(skipped_errors)
|
74
|
+
@traffic_control = traffic_control
|
75
|
+
@traffic_recovery = traffic_recovery
|
76
|
+
end
|
77
|
+
|
78
|
+
# @param other [any]
|
79
|
+
# @return [Boolean]
|
80
|
+
def ==(other)
|
81
|
+
other.is_a?(self.class) && to_h == other.to_h
|
82
|
+
end
|
83
|
+
|
84
|
+
# @param error [Exception]
|
85
|
+
# @return [Boolean]
|
86
|
+
def track_error?(error)
|
87
|
+
skip = skipped_errors.any? { |klass| klass === error }
|
88
|
+
track = tracked_errors.any? { |klass| klass === error }
|
89
|
+
|
90
|
+
!skip && track
|
91
|
+
end
|
92
|
+
|
93
|
+
# Updates the configuration with new settings and returns a new instance.
|
94
|
+
#
|
95
|
+
# @return [Stoplight::Light::Config]
|
96
|
+
def with(**settings)
|
97
|
+
self.class.new(**to_h.merge(settings))
|
98
|
+
end
|
99
|
+
|
100
|
+
# @return [Hash]
|
101
|
+
def to_h
|
102
|
+
{
|
103
|
+
cool_off_time:,
|
104
|
+
data_store:,
|
105
|
+
error_notifier:,
|
106
|
+
name:,
|
107
|
+
notifiers:,
|
108
|
+
threshold:,
|
109
|
+
window_size:,
|
110
|
+
tracked_errors:,
|
111
|
+
skipped_errors:,
|
112
|
+
traffic_control:,
|
113
|
+
traffic_recovery:
|
114
|
+
}
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "forwardable"
|
4
|
+
|
5
|
+
module Stoplight
|
6
|
+
class Light
|
7
|
+
# Implements light configuration behavior
|
8
|
+
module ConfigurationBuilderInterface
|
9
|
+
# Configures data store to be used with this circuit breaker
|
10
|
+
#
|
11
|
+
# @example
|
12
|
+
# Stoplight('example')
|
13
|
+
# .with_data_store(Stoplight::DataStore::Memory.new)
|
14
|
+
#
|
15
|
+
# @param data_store [DataStore::Base]
|
16
|
+
# @return [Stoplight::Light]
|
17
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
18
|
+
def with_data_store(data_store)
|
19
|
+
reconfigure(config.with(data_store: data_store))
|
20
|
+
end
|
21
|
+
|
22
|
+
# Configures cool off time. Stoplight automatically tries to recover
|
23
|
+
# from the red state after the cool off time.
|
24
|
+
#
|
25
|
+
# @example
|
26
|
+
# Stoplight('example')
|
27
|
+
# .cool_off_time(60)
|
28
|
+
#
|
29
|
+
# @param cool_off_time [Numeric] number of seconds
|
30
|
+
# @return [Stoplight::Light]
|
31
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
32
|
+
def with_cool_off_time(cool_off_time)
|
33
|
+
reconfigure(config.with(cool_off_time: cool_off_time))
|
34
|
+
end
|
35
|
+
|
36
|
+
# Configures custom threshold. After this number of failures Stoplight
|
37
|
+
# switches to the red state:
|
38
|
+
#
|
39
|
+
# @example
|
40
|
+
# Stoplight('example')
|
41
|
+
# .with_threshold(5)
|
42
|
+
#
|
43
|
+
# @param threshold [Numeric]
|
44
|
+
# @return [Stoplight::Light]
|
45
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
46
|
+
def with_threshold(threshold)
|
47
|
+
reconfigure(config.with(threshold: threshold))
|
48
|
+
end
|
49
|
+
|
50
|
+
# Configures custom window size which Stoplight uses to count failures. For example,
|
51
|
+
#
|
52
|
+
# @example
|
53
|
+
# Stoplight('example')
|
54
|
+
# .with_threshold(5)
|
55
|
+
# .with_window_size(60)
|
56
|
+
#
|
57
|
+
# The above example will turn to red light only when 5 errors happen
|
58
|
+
# within 60 seconds period.
|
59
|
+
#
|
60
|
+
# @param window_size [Numeric] number of seconds
|
61
|
+
# @return [Stoplight::Light]
|
62
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
63
|
+
def with_window_size(window_size)
|
64
|
+
reconfigure(config.with(window_size: window_size))
|
65
|
+
end
|
66
|
+
|
67
|
+
# Configures custom notifier
|
68
|
+
#
|
69
|
+
# @example
|
70
|
+
# io = StringIO.new
|
71
|
+
# notifier = Stoplight::Notifier::IO.new(io)
|
72
|
+
# Stoplight('example')
|
73
|
+
# .with_notifiers([notifier])
|
74
|
+
#
|
75
|
+
# @param notifiers [Array<Notifier::Base>]
|
76
|
+
# @return [Stoplight::Light]
|
77
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
78
|
+
def with_notifiers(notifiers)
|
79
|
+
reconfigure(config.with(notifiers: notifiers))
|
80
|
+
end
|
81
|
+
|
82
|
+
# @param error_notifier [Proc]
|
83
|
+
# @return [Stoplight::Light]
|
84
|
+
# @api private
|
85
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
86
|
+
def with_error_notifier(&error_notifier)
|
87
|
+
reconfigure(config.with(error_notifier: error_notifier))
|
88
|
+
end
|
89
|
+
|
90
|
+
# Configures a custom list of tracked errors that counts toward the threshold.
|
91
|
+
#
|
92
|
+
# @example
|
93
|
+
# light = Stoplight('example')
|
94
|
+
# .with_tracked_errors(TimeoutError, NetworkError)
|
95
|
+
# light.run { call_external_service }
|
96
|
+
#
|
97
|
+
# In the example above, the +TimeoutError+ and +NetworkError+ exceptions
|
98
|
+
# will be counted towards the threshold for moving the circuit breaker into the red state.
|
99
|
+
# If not configured, the default tracked error is +StandardError+.
|
100
|
+
#
|
101
|
+
# @param tracked_errors [Array<StandardError>]
|
102
|
+
# @return [Stoplight::Light]
|
103
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
104
|
+
def with_tracked_errors(*tracked_errors)
|
105
|
+
reconfigure(config.with(tracked_errors: tracked_errors.dup.freeze))
|
106
|
+
end
|
107
|
+
|
108
|
+
# Configures a custom list of skipped errors that do not count toward the threshold.
|
109
|
+
# Typically, such errors does not represent a real failure and handled somewhere else
|
110
|
+
# in the code.
|
111
|
+
#
|
112
|
+
# @example
|
113
|
+
# light = Stoplight('example')
|
114
|
+
# .with_skipped_errors(ActiveRecord::RecordNotFound)
|
115
|
+
# light.run { User.find(123) }
|
116
|
+
#
|
117
|
+
# In the example above, the +ActiveRecord::RecordNotFound+ doesn't
|
118
|
+
# move the circuit breaker into the red state.
|
119
|
+
#
|
120
|
+
# @param skipped_errors [Array<Exception>]
|
121
|
+
# @return [Stoplight::Light]
|
122
|
+
# @deprecated consider using +Light#with+ for reconfiguration
|
123
|
+
def with_skipped_errors(*skipped_errors)
|
124
|
+
reconfigure(config.with(skipped_errors: skipped_errors))
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Stoplight
|
4
|
+
class Light
|
5
|
+
# Defines how the light executes when it is green.
|
6
|
+
#
|
7
|
+
# This strategy clears failures after successful execution and handles errors
|
8
|
+
# by either raising them or invoking a fallback if provided.
|
9
|
+
#
|
10
|
+
# @api private
|
11
|
+
class GreenRunStrategy < RunStrategy
|
12
|
+
# Executes the provided code block when the light is in the green state.
|
13
|
+
#
|
14
|
+
# @param fallback [Proc, nil] A fallback proc to execute in case of an error.
|
15
|
+
# @yield The code block to execute.
|
16
|
+
# @return [Object] The result of the code block if successful.
|
17
|
+
# @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
|
18
|
+
def execute(fallback, &code)
|
19
|
+
# TODO: Consider implementing sampling rate to limit the memory footprint
|
20
|
+
code.call.tap { record_success }
|
21
|
+
rescue => error
|
22
|
+
if config.track_error?(error)
|
23
|
+
record_error(error)
|
24
|
+
|
25
|
+
if fallback
|
26
|
+
fallback.call(error)
|
27
|
+
else
|
28
|
+
raise
|
29
|
+
end
|
30
|
+
else
|
31
|
+
# User chose to not track the error, so we record it as a success
|
32
|
+
record_success
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private def record_error(error)
|
38
|
+
failure = Stoplight::Failure.from_error(error)
|
39
|
+
metadata = data_store.record_failure(config, failure)
|
40
|
+
|
41
|
+
if config.traffic_control.stop_traffic?(config, metadata) && data_store.transition_to_color(config, Color::RED)
|
42
|
+
config.notifiers.each do |notifier|
|
43
|
+
notifier.notify(config, Color::GREEN, Color::RED, error)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
private def record_success
|
49
|
+
data_store.record_success(config)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Stoplight
|
4
|
+
class Light
|
5
|
+
# Defines how the light executes when it is red.
|
6
|
+
#
|
7
|
+
# This strategy prevents execution of the code block and either raises an error
|
8
|
+
# or invokes a fallback if provided.
|
9
|
+
#
|
10
|
+
# @api private
|
11
|
+
class RedRunStrategy < RunStrategy
|
12
|
+
# Executes the fallback proc when the light is in the red state.
|
13
|
+
#
|
14
|
+
# @param fallback [Proc, nil] A fallback proc to execute instead of the code block.
|
15
|
+
# @return [Object, nil] The result of the fallback proc if provided.
|
16
|
+
# @raise [Stoplight::Error::RedLight] Raises an error if no fallback is provided.
|
17
|
+
def execute(fallback)
|
18
|
+
if fallback
|
19
|
+
fallback.call(nil)
|
20
|
+
else
|
21
|
+
raise Error::RedLight, config.name
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Stoplight
|
4
|
+
class Light
|
5
|
+
# Represents an abstract strategy for running a light's operations.
|
6
|
+
# Every new strategy should be a child of this class.
|
7
|
+
#
|
8
|
+
# @api private
|
9
|
+
# @abstract
|
10
|
+
class RunStrategy
|
11
|
+
# @!attribute [r] config
|
12
|
+
# @return [Stoplight::Light::Config] The configuration for the light.
|
13
|
+
private attr_reader :config
|
14
|
+
|
15
|
+
# @!attribute [r] data_store
|
16
|
+
# @return [Stoplight::DataStore::Base] The data store associated with the light.
|
17
|
+
private attr_reader :data_store
|
18
|
+
|
19
|
+
# @param config [Stoplight::Light::Config] The configuration for the light.
|
20
|
+
def initialize(config)
|
21
|
+
@config = config
|
22
|
+
@data_store = config.data_store
|
23
|
+
end
|
24
|
+
|
25
|
+
def execute(fallback, &code)
|
26
|
+
raise NotImplementedError, "Subclasses must implement the execute method"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Stoplight
|
4
|
+
class Light
|
5
|
+
# Defines how the light executes when it is yellow.
|
6
|
+
#
|
7
|
+
# This strategy clears failures after successful execution and notifies
|
8
|
+
# about color switch from Red to Green. It also handles errors by either
|
9
|
+
# raising them or invoking a fallback if provided.
|
10
|
+
#
|
11
|
+
# @api private
|
12
|
+
class YellowRunStrategy < RunStrategy
|
13
|
+
# Executes the provided code block when the light is in the yellow state.
|
14
|
+
#
|
15
|
+
# @param fallback [Proc, nil] A fallback proc to execute in case of an error.
|
16
|
+
# @yield The code block to execute.
|
17
|
+
# @return [Object] The result of the code block if successful.
|
18
|
+
# @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
|
19
|
+
def execute(fallback, &code)
|
20
|
+
# TODO: We need to employ a probabilistic approach here to avoid "thundering herd" problem
|
21
|
+
code.call.tap { record_recovery_probe_success }
|
22
|
+
rescue => error
|
23
|
+
if config.track_error?(error)
|
24
|
+
record_recovery_probe_failure(error)
|
25
|
+
|
26
|
+
if fallback
|
27
|
+
fallback.call(error)
|
28
|
+
else
|
29
|
+
raise
|
30
|
+
end
|
31
|
+
else
|
32
|
+
record_recovery_probe_success
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private def record_recovery_probe_success
|
38
|
+
metadata = data_store.record_recovery_probe_success(config)
|
39
|
+
|
40
|
+
recover(metadata)
|
41
|
+
end
|
42
|
+
|
43
|
+
private def record_recovery_probe_failure(error)
|
44
|
+
failure = Failure.from_error(error)
|
45
|
+
metadata = data_store.record_recovery_probe_failure(config, failure)
|
46
|
+
|
47
|
+
recover(metadata)
|
48
|
+
end
|
49
|
+
|
50
|
+
private def recover(metadata)
|
51
|
+
recovery_result = config.traffic_recovery.determine_color(config, metadata)
|
52
|
+
|
53
|
+
case recovery_result
|
54
|
+
when Color::GREEN
|
55
|
+
if data_store.transition_to_color(config, Color::GREEN)
|
56
|
+
config.notifiers.each do |notifier|
|
57
|
+
notifier.notify(config, Color::YELLOW, Color::GREEN, nil)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
when Color::YELLOW
|
61
|
+
if data_store.transition_to_color(config, Color::YELLOW)
|
62
|
+
config.notifiers.each do |notifier|
|
63
|
+
notifier.notify(config, Color::GREEN, Color::YELLOW, nil)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
when Color::RED
|
67
|
+
if data_store.transition_to_color(config, Color::RED)
|
68
|
+
config.notifiers.each do |notifier|
|
69
|
+
notifier.notify(config, Color::YELLOW, Color::RED, nil)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
else
|
73
|
+
raise "recovery strategy returned an expected color: #{recovery_result}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|