stoplight 5.5.0 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/stoplight/admin/actions/remove.rb +23 -0
- data/lib/stoplight/admin/dependencies.rb +5 -0
- data/lib/stoplight/admin/lights_repository.rb +12 -3
- data/lib/stoplight/admin/views/_card.erb +13 -1
- data/lib/stoplight/admin.rb +8 -0
- data/lib/stoplight/domain/data_store.rb +42 -6
- data/lib/stoplight/domain/failure.rb +2 -0
- data/lib/stoplight/domain/light.rb +7 -8
- data/lib/stoplight/domain/metrics.rb +85 -0
- data/lib/stoplight/domain/{metadata.rb → state_snapshot.rb} +29 -37
- data/lib/stoplight/domain/strategies/green_run_strategy.rb +2 -2
- data/lib/stoplight/domain/strategies/red_run_strategy.rb +3 -3
- data/lib/stoplight/domain/strategies/run_strategy.rb +2 -2
- data/lib/stoplight/domain/strategies/yellow_run_strategy.rb +7 -6
- data/lib/stoplight/domain/tracker/recovery_probe.rb +9 -6
- data/lib/stoplight/domain/tracker/request.rb +5 -4
- data/lib/stoplight/domain/traffic_control/base.rb +5 -5
- data/lib/stoplight/domain/traffic_control/consecutive_errors.rb +3 -7
- data/lib/stoplight/domain/traffic_control/error_rate.rb +3 -3
- data/lib/stoplight/domain/traffic_recovery/base.rb +6 -5
- data/lib/stoplight/domain/traffic_recovery/consecutive_successes.rb +8 -6
- data/lib/stoplight/infrastructure/data_store/memory/metrics.rb +27 -0
- data/lib/stoplight/infrastructure/data_store/memory/state.rb +21 -0
- data/lib/stoplight/infrastructure/data_store/memory.rb +125 -123
- data/lib/stoplight/infrastructure/data_store/redis/get_metrics.lua +26 -0
- data/lib/stoplight/infrastructure/data_store/redis/lua.rb +1 -1
- data/lib/stoplight/infrastructure/data_store/redis.rb +115 -40
- data/lib/stoplight/version.rb +1 -1
- data/lib/stoplight/wiring/fail_safe_data_store.rb +27 -3
- metadata +7 -3
- data/lib/stoplight/infrastructure/data_store/redis/get_metadata.lua +0 -38
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f540f78d771a711da32d2e50cc150c4072cf805e4ef7e52e0b5ea07dd2435fea
|
|
4
|
+
data.tar.gz: 2b859ca65577026db5e1e5af4bc2900e178edc0e230b1cfc75728ffcadc9efc8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 88eaa1d335e4b956cc354d4c8295e46acd6f4dbb0188cc4cd5ecef99a13c4cf666e09e9c2d4c4f926f2ed0c5a582c9283341de028dc4489645cbde256d805504
|
|
7
|
+
data.tar.gz: d94325879c511730d6b53b40d13fed577e10119813ec3be5f18f92caccf42d1a886d8c0fe8d6c887ff3d96610f7ac2c53e3d992d25e257cf38fea33795260272
|
data/README.md
CHANGED
|
@@ -534,7 +534,7 @@ class ApplicationController < ActionController::Base
|
|
|
534
534
|
|
|
535
535
|
def stoplight(&block)
|
|
536
536
|
Stoplight("#{params[:controller]}##{params[:action]}")
|
|
537
|
-
.run(-> { render(nothing: true, status: :service_unavailable) }, &block)
|
|
537
|
+
.run(-> (*) { render(nothing: true, status: :service_unavailable) }, &block)
|
|
538
538
|
end
|
|
539
539
|
end
|
|
540
540
|
```
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Stoplight
|
|
4
|
+
class Admin
|
|
5
|
+
module Actions
|
|
6
|
+
# This action removes a light's metadata from Redis
|
|
7
|
+
class Remove < Action
|
|
8
|
+
# @param params [Hash] query parameters
|
|
9
|
+
# @return [void]
|
|
10
|
+
def call(params)
|
|
11
|
+
light_names(params).each do |name|
|
|
12
|
+
lights_repository.remove(name)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
private def light_names(params)
|
|
17
|
+
Array(params[:names])
|
|
18
|
+
.map { |name| CGI.unescape(name) }
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -45,6 +45,11 @@ module Stoplight
|
|
|
45
45
|
def green_all_action
|
|
46
46
|
Stoplight::Admin::Actions::LockAllGreen.new(lights_repository: lights_repository)
|
|
47
47
|
end
|
|
48
|
+
|
|
49
|
+
# @return [Stoplight::Admin::Actions::Remove]
|
|
50
|
+
def remove_action
|
|
51
|
+
Stoplight::Admin::Actions::Remove.new(lights_repository: lights_repository)
|
|
52
|
+
end
|
|
48
53
|
end
|
|
49
54
|
end
|
|
50
55
|
end
|
|
@@ -53,16 +53,25 @@ module Stoplight
|
|
|
53
53
|
build_light(name).unlock
|
|
54
54
|
end
|
|
55
55
|
|
|
56
|
+
# @param name [String] removes light metadata by its name
|
|
57
|
+
# @return [void]
|
|
58
|
+
def remove(name)
|
|
59
|
+
light = build_light(name)
|
|
60
|
+
|
|
61
|
+
data_store.delete_light(light.config)
|
|
62
|
+
end
|
|
63
|
+
|
|
56
64
|
private def load_light(name)
|
|
57
65
|
light = build_light(name)
|
|
58
66
|
# failures, state
|
|
59
|
-
|
|
67
|
+
state_snapshot = data_store.get_state_snapshot(light.config)
|
|
68
|
+
metrics = data_store.get_metrics(light.config)
|
|
60
69
|
|
|
61
70
|
Light.new(
|
|
62
71
|
name: name,
|
|
63
72
|
color: light.color,
|
|
64
|
-
state:
|
|
65
|
-
failures: [
|
|
73
|
+
state: state_snapshot.locked_state,
|
|
74
|
+
failures: [metrics.last_error].compact
|
|
66
75
|
)
|
|
67
76
|
end
|
|
68
77
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
<div class="max-w-xl mr-5 p-6 border border-gray-200 rounded-lg shadow-sm dark:bg-gray-800 dark:border-gray-700">
|
|
2
2
|
<div class="flex items-center">
|
|
3
|
-
<% light_name =
|
|
3
|
+
<% light_name = CGI.escape(light.name) %>
|
|
4
4
|
|
|
5
5
|
<div class="relative">
|
|
6
6
|
<div class="relative inline-flex items-center justify-center w-10 h-10 overflow-hidden bg-<%= color %>-100 rounded-full dark:bg-<%= color %>-600">
|
|
@@ -83,6 +83,18 @@
|
|
|
83
83
|
Lock Green
|
|
84
84
|
</a>
|
|
85
85
|
</li>
|
|
86
|
+
<li>
|
|
87
|
+
<a href="<%= url("/remove?names=#{light_name}") %>" data-turbo-method="post" data-turbo-confirm="Are you sure you want to remove this light?" class="flex items-center py-2 px-4 text-sm hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">
|
|
88
|
+
<svg class="flex w-4 h-4 me-1.5 text-red-600 shrink-0" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
|
89
|
+
<polyline points="3 6 5 6 21 6"/>
|
|
90
|
+
<path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"/>
|
|
91
|
+
<path d="M10 11v6"/>
|
|
92
|
+
<path d="M14 11v6"/>
|
|
93
|
+
<path d="M9 6V4a2 2 0 0 1 2-2h2a2 2 0 0 1 2 2v2"/>
|
|
94
|
+
</svg>
|
|
95
|
+
Remove
|
|
96
|
+
</a>
|
|
97
|
+
</li>
|
|
86
98
|
</ul>
|
|
87
99
|
</div>
|
|
88
100
|
</div>
|
data/lib/stoplight/admin.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "cgi" # Ruby 3.2 needs this
|
|
4
|
+
|
|
3
5
|
begin
|
|
4
6
|
require "sinatra/base"
|
|
5
7
|
require "sinatra/json"
|
|
@@ -63,5 +65,11 @@ module Stoplight
|
|
|
63
65
|
|
|
64
66
|
redirect to("/")
|
|
65
67
|
end
|
|
68
|
+
|
|
69
|
+
post "/remove" do
|
|
70
|
+
dependencies.remove_action.call(params)
|
|
71
|
+
|
|
72
|
+
redirect to("/")
|
|
73
|
+
end
|
|
66
74
|
end
|
|
67
75
|
end
|
|
@@ -14,11 +14,37 @@ module Stoplight
|
|
|
14
14
|
raise NotImplementedError
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
# Retrieves
|
|
17
|
+
# Retrieves metrics for a specific light configuration.
|
|
18
|
+
#
|
|
19
|
+
# @param config [Stoplight::Domain::Config]
|
|
20
|
+
# @return [Stoplight::Domain::Metrics]
|
|
21
|
+
def get_metrics(config)
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Retrieves recovery metrics for a specific light configuration.
|
|
26
|
+
#
|
|
27
|
+
# @param config [Stoplight::Domain::Config]
|
|
28
|
+
# @return [Stoplight::Domain::Metrics]
|
|
29
|
+
def get_recovery_metrics(config)
|
|
30
|
+
raise NotImplementedError
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Retrieves State Snapshot for a specific light configuration.
|
|
34
|
+
#
|
|
35
|
+
# @param config [Stoplight::Domain::Config] The light configuration.
|
|
36
|
+
# @return [Stoplight::Domain::StateSnapshot]
|
|
37
|
+
def get_state_snapshot(config)
|
|
38
|
+
raise NotImplementedError
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Clears windowed metrics (successes/errors) to prevent
|
|
42
|
+
# stale failures from before recovery from affecting post-recovery decisions.
|
|
43
|
+
# Consecutive counts are intentionally preserved as they track current streaks.
|
|
18
44
|
#
|
|
19
45
|
# @param config [Stoplight::Domain::Config] The light configuration.
|
|
20
|
-
# @return [
|
|
21
|
-
def
|
|
46
|
+
# @return [void]
|
|
47
|
+
def clear_windowed_metrics(config)
|
|
22
48
|
raise NotImplementedError
|
|
23
49
|
end
|
|
24
50
|
|
|
@@ -26,7 +52,7 @@ module Stoplight
|
|
|
26
52
|
#
|
|
27
53
|
# @param config [Stoplight::Domain::Config]
|
|
28
54
|
# @param exception [Exception]
|
|
29
|
-
# @return [
|
|
55
|
+
# @return [void]
|
|
30
56
|
def record_failure(config, exception)
|
|
31
57
|
raise NotImplementedError
|
|
32
58
|
end
|
|
@@ -43,7 +69,7 @@ module Stoplight
|
|
|
43
69
|
#
|
|
44
70
|
# @param config [Stoplight::Domain::Config]
|
|
45
71
|
# @param failure [Failure]
|
|
46
|
-
# @return [
|
|
72
|
+
# @return [void]
|
|
47
73
|
def record_recovery_probe_failure(config, failure)
|
|
48
74
|
raise NotImplementedError
|
|
49
75
|
end
|
|
@@ -51,7 +77,7 @@ module Stoplight
|
|
|
51
77
|
# Records a successful recovery probe for a specific light configuration.
|
|
52
78
|
#
|
|
53
79
|
# @param config [Stoplight::Domain::Config]
|
|
54
|
-
# @return [
|
|
80
|
+
# @return [void]
|
|
55
81
|
def record_recovery_probe_success(config)
|
|
56
82
|
raise NotImplementedError
|
|
57
83
|
end
|
|
@@ -88,6 +114,16 @@ module Stoplight
|
|
|
88
114
|
def transition_to_color(config, color)
|
|
89
115
|
raise NotImplementedError
|
|
90
116
|
end
|
|
117
|
+
|
|
118
|
+
# Deletes metadata (and related persistent state) for the given light.
|
|
119
|
+
#
|
|
120
|
+
# Implementations may choose to only remove metadata; metrics may expire via TTL.
|
|
121
|
+
#
|
|
122
|
+
# @param config [Stoplight::Domain::Config]
|
|
123
|
+
# @return [void]
|
|
124
|
+
def delete_light(config)
|
|
125
|
+
raise NotImplementedError
|
|
126
|
+
end
|
|
91
127
|
end
|
|
92
128
|
# :nocov:
|
|
93
129
|
end
|
|
@@ -57,7 +57,7 @@ module Stoplight
|
|
|
57
57
|
#
|
|
58
58
|
# @return [String]
|
|
59
59
|
def state
|
|
60
|
-
|
|
60
|
+
state_snapshot.locked_state
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
# Returns current color:
|
|
@@ -71,7 +71,7 @@ module Stoplight
|
|
|
71
71
|
#
|
|
72
72
|
# @return [String] returns current light color
|
|
73
73
|
def color
|
|
74
|
-
|
|
74
|
+
state_snapshot.color
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
# Runs the given block of code with this circuit breaker
|
|
@@ -91,9 +91,9 @@ module Stoplight
|
|
|
91
91
|
def run(fallback = nil, &code)
|
|
92
92
|
raise ArgumentError, "nothing to run. Please, pass a block into `Light#run`" unless block_given?
|
|
93
93
|
|
|
94
|
-
|
|
95
|
-
strategy = state_strategy_factory(
|
|
96
|
-
strategy.execute(fallback,
|
|
94
|
+
state_snapshot.then do |state_snapshot|
|
|
95
|
+
strategy = state_strategy_factory(state_snapshot.color)
|
|
96
|
+
strategy.execute(fallback, state_snapshot:, &code)
|
|
97
97
|
end
|
|
98
98
|
end
|
|
99
99
|
|
|
@@ -189,9 +189,8 @@ module Stoplight
|
|
|
189
189
|
end
|
|
190
190
|
end
|
|
191
191
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
data_store.get_metadata(config)
|
|
192
|
+
def state_snapshot
|
|
193
|
+
data_store.get_state_snapshot(config)
|
|
195
194
|
end
|
|
196
195
|
end
|
|
197
196
|
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Stoplight
|
|
4
|
+
module Domain
|
|
5
|
+
# Request metrics over a given window.
|
|
6
|
+
#
|
|
7
|
+
# @!attribute successes
|
|
8
|
+
# A number of successes withing requested window. Zero for non-windowed metrics
|
|
9
|
+
# @return [Integer]
|
|
10
|
+
#
|
|
11
|
+
# @!attribute errors
|
|
12
|
+
# A number of errors withing requested window. Zero for non-windowed metrics
|
|
13
|
+
# @return [Integer]
|
|
14
|
+
#
|
|
15
|
+
# @!attribute total_consecutive_errors
|
|
16
|
+
# A total number of consecutive errors
|
|
17
|
+
# @return [Integer]
|
|
18
|
+
#
|
|
19
|
+
# @!attribute total_consecutive_successes
|
|
20
|
+
# A total number of consecutive successes
|
|
21
|
+
# @return [Integer]
|
|
22
|
+
#
|
|
23
|
+
# @!attribute last_error
|
|
24
|
+
# @return [Stoplight::Domain::Failure, nil]
|
|
25
|
+
#
|
|
26
|
+
# @!attribute last_success_at
|
|
27
|
+
# @return [Time, nil]
|
|
28
|
+
#
|
|
29
|
+
# @api private
|
|
30
|
+
Metrics = Data.define(
|
|
31
|
+
:successes,
|
|
32
|
+
:errors,
|
|
33
|
+
:total_consecutive_errors,
|
|
34
|
+
:total_consecutive_successes,
|
|
35
|
+
:last_error,
|
|
36
|
+
:last_success_at
|
|
37
|
+
) do
|
|
38
|
+
# A number of consecutive errors withing requested window
|
|
39
|
+
#
|
|
40
|
+
# @return [Integer]
|
|
41
|
+
def consecutive_errors
|
|
42
|
+
if errors # we effectively check if this is windowed metrics
|
|
43
|
+
[total_consecutive_errors, errors].min
|
|
44
|
+
else
|
|
45
|
+
total_consecutive_errors
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A number of consecutive successes withing requested window
|
|
50
|
+
#
|
|
51
|
+
def consecutive_successes
|
|
52
|
+
if successes # we effectively check if this is windowed metrics
|
|
53
|
+
[total_consecutive_successes, successes].min
|
|
54
|
+
else
|
|
55
|
+
total_consecutive_successes
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Calculates the error rate based on the number of successes and errors.
|
|
60
|
+
#
|
|
61
|
+
# @return [Float]
|
|
62
|
+
def error_rate
|
|
63
|
+
return unless requests # we effectively check if this is windowed metrics
|
|
64
|
+
|
|
65
|
+
if (successes + errors).zero?
|
|
66
|
+
0.0
|
|
67
|
+
else
|
|
68
|
+
errors.fdiv(successes + errors)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @return [Integer]
|
|
73
|
+
def requests
|
|
74
|
+
if successes && errors # we effectively check if this is windowed metrics
|
|
75
|
+
successes + errors
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @return [Time, nil]
|
|
80
|
+
def last_error_at
|
|
81
|
+
last_error&.time
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -2,41 +2,39 @@
|
|
|
2
2
|
|
|
3
3
|
module Stoplight
|
|
4
4
|
module Domain
|
|
5
|
-
#
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
5
|
+
# @!attribute breached_at
|
|
6
|
+
# The time when the light became red (breached threshold)
|
|
7
|
+
# @return [Time, nil]
|
|
8
|
+
#
|
|
9
|
+
# @!attribute locked_state
|
|
10
|
+
# @return [State::UNLOCKED | State::LOCKED_GREEN | State::LOCKED_RED]
|
|
11
|
+
#
|
|
12
|
+
# @!attribute recovery_scheduled_after
|
|
13
|
+
# When Light transitions to RED, it schedules recovery after the Cool Off Time.
|
|
14
|
+
# @return [Time, nil]
|
|
15
|
+
#
|
|
16
|
+
# @!attribute recovery_started_at
|
|
17
|
+
# When in YELLOW state, this time indicates the time of transitioning to YELLOW
|
|
18
|
+
# @return [Time, nil]
|
|
19
|
+
#
|
|
20
|
+
# @!attribute time
|
|
21
|
+
# The time when the snapshot was taken
|
|
22
|
+
# @return [Time]
|
|
23
|
+
#
|
|
24
|
+
StateSnapshot = Data.define(
|
|
16
25
|
:breached_at,
|
|
17
26
|
:locked_state,
|
|
18
27
|
:recovery_scheduled_after,
|
|
19
28
|
:recovery_started_at,
|
|
20
|
-
:
|
|
21
|
-
:current_time
|
|
29
|
+
:time
|
|
22
30
|
) do
|
|
23
|
-
# YELLOW color could be entered implicitly through a timeout
|
|
24
|
-
# and explicitly through a transition.
|
|
25
|
-
#
|
|
26
|
-
# This method indicates whether the recovery has already started explicitly
|
|
27
|
-
#
|
|
28
|
-
# @return [Boolean]
|
|
29
|
-
def recovery_started?
|
|
30
|
-
recovery_started_at && recovery_started_at <= current_time
|
|
31
|
-
end
|
|
32
|
-
|
|
33
31
|
# @return [String] one of +Color::GREEN+, +Color::RED+, or +Color::YELLOW+
|
|
34
32
|
def color
|
|
35
33
|
if locked_state == State::LOCKED_GREEN
|
|
36
34
|
Color::GREEN
|
|
37
35
|
elsif locked_state == State::LOCKED_RED
|
|
38
36
|
Color::RED
|
|
39
|
-
elsif (recovery_scheduled_after && recovery_scheduled_after <
|
|
37
|
+
elsif (recovery_scheduled_after && recovery_scheduled_after < time) || recovery_started_at
|
|
40
38
|
Color::YELLOW
|
|
41
39
|
elsif breached_at
|
|
42
40
|
Color::RED
|
|
@@ -45,20 +43,14 @@ module Stoplight
|
|
|
45
43
|
end
|
|
46
44
|
end
|
|
47
45
|
|
|
48
|
-
#
|
|
46
|
+
# YELLOW color could be entered implicitly through a timeout
|
|
47
|
+
# and explicitly through a transition.
|
|
49
48
|
#
|
|
50
|
-
#
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
errors.fdiv(successes + errors)
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# @return [Integer]
|
|
60
|
-
def requests
|
|
61
|
-
successes + errors
|
|
49
|
+
# This method indicates whether the recovery has already started explicitly
|
|
50
|
+
#
|
|
51
|
+
# @return [Boolean]
|
|
52
|
+
def recovery_started?
|
|
53
|
+
recovery_started_at && recovery_started_at <= time
|
|
62
54
|
end
|
|
63
55
|
end
|
|
64
56
|
end
|
|
@@ -28,11 +28,11 @@ module Stoplight
|
|
|
28
28
|
# Executes the provided code block when the light is in the green state.
|
|
29
29
|
#
|
|
30
30
|
# @param fallback [Proc, nil] A fallback proc to execute in case of an error.
|
|
31
|
-
# @param
|
|
31
|
+
# @param state_snapshot [Stoplight::Domain::StateSnapshot]
|
|
32
32
|
# @yield The code block to execute.
|
|
33
33
|
# @return [Object] The result of the code block if successful.
|
|
34
34
|
# @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
|
|
35
|
-
def execute(fallback,
|
|
35
|
+
def execute(fallback, state_snapshot:, &code)
|
|
36
36
|
# TODO: Consider implementing sampling rate to limit the memory footprint
|
|
37
37
|
code.call.tap { record_success }
|
|
38
38
|
rescue => error
|
|
@@ -21,17 +21,17 @@ module Stoplight
|
|
|
21
21
|
# Executes the fallback proc when the light is in the red state.
|
|
22
22
|
#
|
|
23
23
|
# @param fallback [Proc, nil] A fallback proc to execute instead of the code block.
|
|
24
|
-
# @param
|
|
24
|
+
# @param state_snapshot [Stoplight::Domain::StateSnapshot]
|
|
25
25
|
# @return [Object, nil] The result of the fallback proc if provided.
|
|
26
26
|
# @raise [Stoplight::Error::RedLight] Raises an error if no fallback is provided.
|
|
27
|
-
def execute(fallback,
|
|
27
|
+
def execute(fallback, state_snapshot:)
|
|
28
28
|
if fallback
|
|
29
29
|
fallback.call(nil)
|
|
30
30
|
else
|
|
31
31
|
raise Error::RedLight.new(
|
|
32
32
|
config.name,
|
|
33
33
|
cool_off_time: config.cool_off_time,
|
|
34
|
-
retry_after:
|
|
34
|
+
retry_after: state_snapshot.recovery_scheduled_after
|
|
35
35
|
)
|
|
36
36
|
end
|
|
37
37
|
end
|
|
@@ -10,9 +10,9 @@ module Stoplight
|
|
|
10
10
|
# @abstract
|
|
11
11
|
class RunStrategy
|
|
12
12
|
# @param fallback [Proc, nil] A fallback proc to execute in case of an error.
|
|
13
|
-
# @param
|
|
13
|
+
# @param state_snapshot [Stoplight::Domain::StateSnapshot]
|
|
14
14
|
# :nocov:
|
|
15
|
-
def execute(fallback,
|
|
15
|
+
def execute(fallback, state_snapshot:, &code)
|
|
16
16
|
raise NotImplementedError, "Subclasses must implement the execute method"
|
|
17
17
|
end
|
|
18
18
|
# :nocov:
|
|
@@ -41,12 +41,12 @@ module Stoplight
|
|
|
41
41
|
# Executes the provided code block when the light is in the yellow state.
|
|
42
42
|
#
|
|
43
43
|
# @param fallback [Proc, nil] A fallback proc to execute in case of an error.
|
|
44
|
-
# @param
|
|
44
|
+
# @param state_snapshot [Stoplight::Domain::StateSnapshot]
|
|
45
45
|
# @yield The code block to execute.
|
|
46
46
|
# @return [Object] The result of the code block if successful.
|
|
47
47
|
# @raise [Exception] Re-raises the error if it is not tracked or no fallback is provided.
|
|
48
|
-
def execute(fallback,
|
|
49
|
-
enter_recovery(
|
|
48
|
+
def execute(fallback, state_snapshot:, &code)
|
|
49
|
+
enter_recovery(state_snapshot)
|
|
50
50
|
# TODO: We need to employ a probabilistic approach here to avoid "thundering herd" problem
|
|
51
51
|
code.call.tap { record_recovery_probe_success }
|
|
52
52
|
rescue => error
|
|
@@ -72,12 +72,13 @@ module Stoplight
|
|
|
72
72
|
request_tracker.record_failure(error)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
# @param
|
|
75
|
+
# @param state_snapshot [Stoplight::Domain::StateSnapshot]
|
|
76
76
|
# @return [void]
|
|
77
|
-
private def enter_recovery(
|
|
78
|
-
return if
|
|
77
|
+
private def enter_recovery(state_snapshot)
|
|
78
|
+
return if state_snapshot.recovery_started?
|
|
79
79
|
|
|
80
80
|
if data_store.transition_to_color(config, Color::YELLOW)
|
|
81
|
+
data_store.clear_windowed_metrics(config)
|
|
81
82
|
notifiers.each do |notifier|
|
|
82
83
|
notifier.notify(config, Color::RED, Color::YELLOW, nil)
|
|
83
84
|
end
|
|
@@ -33,15 +33,15 @@ module Stoplight
|
|
|
33
33
|
|
|
34
34
|
# @param exception [Exception]
|
|
35
35
|
def record_failure(exception)
|
|
36
|
-
|
|
36
|
+
data_store.record_recovery_probe_failure(config, exception)
|
|
37
37
|
|
|
38
|
-
recover
|
|
38
|
+
recover
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
def record_success
|
|
42
|
-
|
|
42
|
+
data_store.record_recovery_probe_success(config)
|
|
43
43
|
|
|
44
|
-
recover
|
|
44
|
+
recover
|
|
45
45
|
end
|
|
46
46
|
RECOVERY_TRANSITIONS = {
|
|
47
47
|
TrafficRecovery::GREEN => [Color::YELLOW, Color::GREEN],
|
|
@@ -49,8 +49,11 @@ module Stoplight
|
|
|
49
49
|
TrafficRecovery::RED => [Color::YELLOW, Color::RED]
|
|
50
50
|
}.freeze
|
|
51
51
|
|
|
52
|
-
private def recover
|
|
53
|
-
|
|
52
|
+
private def recover
|
|
53
|
+
recovery_metrics = data_store.get_recovery_metrics(config)
|
|
54
|
+
state_snapshot = data_store.get_state_snapshot(config) # TODO: is this really necessary?
|
|
55
|
+
|
|
56
|
+
recovery_result = traffic_recovery.determine_color(config, recovery_metrics, state_snapshot)
|
|
54
57
|
|
|
55
58
|
return if recovery_result == TrafficRecovery::PASS
|
|
56
59
|
|
|
@@ -40,9 +40,10 @@ module Stoplight
|
|
|
40
40
|
# @param exception [Exception]
|
|
41
41
|
# @return [void]
|
|
42
42
|
def record_failure(exception)
|
|
43
|
-
|
|
43
|
+
data_store.record_failure(config, exception)
|
|
44
|
+
metrics = data_store.get_metrics(config)
|
|
44
45
|
|
|
45
|
-
transition_to_red(exception,
|
|
46
|
+
transition_to_red(exception, metrics:)
|
|
46
47
|
end
|
|
47
48
|
|
|
48
49
|
# @return [void]
|
|
@@ -50,8 +51,8 @@ module Stoplight
|
|
|
50
51
|
data_store.record_success(config)
|
|
51
52
|
end
|
|
52
53
|
|
|
53
|
-
private def transition_to_red(exception,
|
|
54
|
-
if traffic_control.stop_traffic?(config,
|
|
54
|
+
private def transition_to_red(exception, metrics:)
|
|
55
|
+
if traffic_control.stop_traffic?(config, metrics)
|
|
55
56
|
transition_and_notify(Color::GREEN, Color::RED, exception)
|
|
56
57
|
end
|
|
57
58
|
end
|
|
@@ -18,11 +18,11 @@ module Stoplight
|
|
|
18
18
|
# end
|
|
19
19
|
# end
|
|
20
20
|
#
|
|
21
|
-
# def stop_traffic?(config,
|
|
22
|
-
# total =
|
|
21
|
+
# def stop_traffic?(config, metrics)
|
|
22
|
+
# total = metrics.successes + metrics.failures
|
|
23
23
|
# return false if total < 10 # Minimum sample size
|
|
24
24
|
#
|
|
25
|
-
# error_rate =
|
|
25
|
+
# error_rate = metrics.failures.fdiv(total)
|
|
26
26
|
# error_rate >= 0.5 # Stop traffic when error rate reaches 50%
|
|
27
27
|
# end
|
|
28
28
|
# end
|
|
@@ -44,10 +44,10 @@ module Stoplight
|
|
|
44
44
|
# current state and metrics.
|
|
45
45
|
#
|
|
46
46
|
# @param config [Stoplight::Domain::Config]
|
|
47
|
-
# @param
|
|
47
|
+
# @param metrics [Stoplight::Domain::Metrics]
|
|
48
48
|
# @return [Boolean] true if traffic should be stopped (rec), false otherwise (green)
|
|
49
49
|
# :nocov:
|
|
50
|
-
def stop_traffic?(config,
|
|
50
|
+
def stop_traffic?(config, metrics)
|
|
51
51
|
raise NotImplementedError
|
|
52
52
|
end
|
|
53
53
|
# :nocov:
|
|
@@ -42,14 +42,10 @@ module Stoplight
|
|
|
42
42
|
# Determines if traffic should be stopped based on failure counts.
|
|
43
43
|
#
|
|
44
44
|
# @param config [Stoplight::Domain::Config]
|
|
45
|
-
# @param
|
|
45
|
+
# @param metrics [Stoplight::Domain::Metrics]
|
|
46
46
|
# @return [Boolean] true if failures have reached the threshold, false otherwise
|
|
47
|
-
def stop_traffic?(config,
|
|
48
|
-
|
|
49
|
-
[metadata.consecutive_errors, metadata.errors].min >= config.threshold
|
|
50
|
-
else
|
|
51
|
-
metadata.consecutive_errors >= config.threshold
|
|
52
|
-
end
|
|
47
|
+
def stop_traffic?(config, metrics)
|
|
48
|
+
metrics.consecutive_errors >= config.threshold
|
|
53
49
|
end
|
|
54
50
|
end
|
|
55
51
|
end
|
|
@@ -40,10 +40,10 @@ module Stoplight
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
# @param config [Stoplight::Domain::Config]
|
|
43
|
-
# @param
|
|
43
|
+
# @param metrics [Stoplight::Domain::Metrics]
|
|
44
44
|
# @return [Boolean]
|
|
45
|
-
def stop_traffic?(config,
|
|
46
|
-
|
|
45
|
+
def stop_traffic?(config, metrics)
|
|
46
|
+
metrics.requests >= min_requests && metrics.error_rate >= config.threshold
|
|
47
47
|
end
|
|
48
48
|
end
|
|
49
49
|
end
|