stoplight 5.4.0 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/lib/stoplight/admin/actions/remove.rb +23 -0
- data/lib/stoplight/admin/dependencies.rb +5 -0
- data/lib/stoplight/admin/lights_repository.rb +12 -3
- data/lib/stoplight/admin/views/_card.erb +13 -1
- data/lib/stoplight/admin/views/layout.erb +3 -3
- data/lib/stoplight/admin.rb +12 -4
- data/lib/stoplight/domain/color.rb +11 -0
- data/lib/stoplight/{config → domain}/compatibility_result.rb +1 -1
- data/lib/stoplight/domain/config.rb +55 -0
- data/lib/stoplight/{data_store/base.rb → domain/data_store.rb} +55 -17
- data/lib/stoplight/domain/error.rb +42 -0
- data/lib/stoplight/domain/failure.rb +44 -0
- data/lib/stoplight/domain/light/configuration_builder_interface.rb +130 -0
- data/lib/stoplight/domain/light.rb +197 -0
- data/lib/stoplight/domain/light_factory.rb +75 -0
- data/lib/stoplight/domain/metrics.rb +85 -0
- data/lib/stoplight/domain/state.rb +11 -0
- data/lib/stoplight/domain/state_snapshot.rb +57 -0
- data/lib/stoplight/{notifier/base.rb → domain/state_transition_notifier.rb} +5 -4
- data/lib/stoplight/domain/strategies/green_run_strategy.rb +69 -0
- data/lib/stoplight/domain/strategies/red_run_strategy.rb +41 -0
- data/lib/stoplight/domain/strategies/run_strategy.rb +27 -0
- data/lib/stoplight/domain/strategies/yellow_run_strategy.rb +99 -0
- data/lib/stoplight/domain/tracker/base.rb +41 -0
- data/lib/stoplight/domain/tracker/recovery_probe.rb +75 -0
- data/lib/stoplight/domain/tracker/request.rb +68 -0
- data/lib/stoplight/domain/traffic_control/base.rb +74 -0
- data/lib/stoplight/domain/traffic_control/consecutive_errors.rb +53 -0
- data/lib/stoplight/domain/traffic_control/error_rate.rb +51 -0
- data/lib/stoplight/domain/traffic_recovery/base.rb +80 -0
- data/lib/stoplight/domain/traffic_recovery/consecutive_successes.rb +72 -0
- data/lib/stoplight/domain/traffic_recovery.rb +13 -0
- data/lib/stoplight/infrastructure/data_store/memory/metrics.rb +27 -0
- data/lib/stoplight/infrastructure/data_store/memory/sliding_window.rb +79 -0
- data/lib/stoplight/infrastructure/data_store/memory/state.rb +21 -0
- data/lib/stoplight/infrastructure/data_store/memory.rb +309 -0
- data/lib/stoplight/infrastructure/data_store/redis/get_metrics.lua +26 -0
- data/lib/stoplight/infrastructure/data_store/redis/lua.rb +25 -0
- data/lib/stoplight/infrastructure/data_store/redis.rb +553 -0
- data/lib/stoplight/infrastructure/dependency_injection/container.rb +249 -0
- data/lib/stoplight/infrastructure/dependency_injection/unresolved_dependency_error.rb +13 -0
- data/lib/stoplight/infrastructure/notifier/generic.rb +90 -0
- data/lib/stoplight/infrastructure/notifier/io.rb +23 -0
- data/lib/stoplight/infrastructure/notifier/logger.rb +21 -0
- data/lib/stoplight/rspec/generic_notifier.rb +1 -1
- data/lib/stoplight/version.rb +1 -1
- data/lib/stoplight/wiring/container.rb +80 -0
- data/lib/stoplight/wiring/default.rb +28 -0
- data/lib/stoplight/{config/user_default_config.rb → wiring/default_configuration.rb} +24 -31
- data/lib/stoplight/wiring/default_factory_builder.rb +25 -0
- data/lib/stoplight/{data_store/fail_safe.rb → wiring/fail_safe_data_store.rb} +49 -14
- data/lib/stoplight/{notifier/fail_safe.rb → wiring/fail_safe_notifier.rb} +22 -13
- data/lib/stoplight/wiring/light/default_config.rb +18 -0
- data/lib/stoplight/wiring/light/system_config.rb +11 -0
- data/lib/stoplight/wiring/light_factory.rb +188 -0
- data/lib/stoplight/wiring/public_api.rb +28 -0
- data/lib/stoplight/wiring/system_container.rb +9 -0
- data/lib/stoplight/wiring/system_light_factory.rb +17 -0
- data/lib/stoplight.rb +38 -28
- metadata +57 -43
- data/lib/stoplight/color.rb +0 -9
- data/lib/stoplight/config/dsl.rb +0 -97
- data/lib/stoplight/config/library_default_config.rb +0 -21
- data/lib/stoplight/config/system_config.rb +0 -10
- data/lib/stoplight/data_store/memory/sliding_window.rb +0 -77
- data/lib/stoplight/data_store/memory.rb +0 -285
- data/lib/stoplight/data_store/redis/get_metadata.lua +0 -38
- data/lib/stoplight/data_store/redis/lua.rb +0 -23
- data/lib/stoplight/data_store/redis.rb +0 -446
- data/lib/stoplight/data_store.rb +0 -6
- data/lib/stoplight/default.rb +0 -30
- data/lib/stoplight/error.rb +0 -39
- data/lib/stoplight/failure.rb +0 -71
- data/lib/stoplight/light/config.rb +0 -112
- data/lib/stoplight/light/configuration_builder_interface.rb +0 -128
- data/lib/stoplight/light/green_run_strategy.rb +0 -54
- data/lib/stoplight/light/red_run_strategy.rb +0 -31
- data/lib/stoplight/light/run_strategy.rb +0 -32
- data/lib/stoplight/light/yellow_run_strategy.rb +0 -94
- data/lib/stoplight/light.rb +0 -191
- data/lib/stoplight/metadata.rb +0 -99
- data/lib/stoplight/notifier/generic.rb +0 -79
- data/lib/stoplight/notifier/io.rb +0 -21
- data/lib/stoplight/notifier/logger.rb +0 -19
- data/lib/stoplight/state.rb +0 -9
- data/lib/stoplight/traffic_control/base.rb +0 -70
- data/lib/stoplight/traffic_control/consecutive_errors.rb +0 -55
- data/lib/stoplight/traffic_control/error_rate.rb +0 -49
- data/lib/stoplight/traffic_recovery/base.rb +0 -75
- data/lib/stoplight/traffic_recovery/consecutive_successes.rb +0 -68
- data/lib/stoplight/traffic_recovery.rb +0 -11
- /data/lib/stoplight/{data_store → infrastructure/data_store}/redis/record_failure.lua +0 -0
- /data/lib/stoplight/{data_store → infrastructure/data_store}/redis/record_success.lua +0 -0
- /data/lib/stoplight/{data_store → infrastructure/data_store}/redis/transition_to_green.lua +0 -0
- /data/lib/stoplight/{data_store → infrastructure/data_store}/redis/transition_to_red.lua +0 -0
- /data/lib/stoplight/{data_store → infrastructure/data_store}/redis/transition_to_yellow.lua +0 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "monitor"
|
|
4
|
+
|
|
5
|
+
module Stoplight
|
|
6
|
+
module Infrastructure
|
|
7
|
+
module DataStore
|
|
8
|
+
# @see +Domain::DataStore+
|
|
9
|
+
class Memory < Domain::DataStore
|
|
10
|
+
include MonitorMixin
|
|
11
|
+
|
|
12
|
+
KEY_SEPARATOR = ":"
|
|
13
|
+
|
|
14
|
+
def initialize
|
|
15
|
+
@errors = Hash.new { |errors, light_name| errors[light_name] = SlidingWindow.new }
|
|
16
|
+
@successes = Hash.new { |successes, light_name| successes[light_name] = SlidingWindow.new }
|
|
17
|
+
|
|
18
|
+
@recovery_probe_errors = Hash.new { |recovery_probe_errors, light_name| recovery_probe_errors[light_name] = SlidingWindow.new }
|
|
19
|
+
@recovery_probe_successes = Hash.new { |recovery_probe_successes, light_name| recovery_probe_successes[light_name] = SlidingWindow.new }
|
|
20
|
+
|
|
21
|
+
@states = Hash.new { |states, light_name| states[light_name] = State.new }
|
|
22
|
+
@metrics = Hash.new { |metrics, light_name| metrics[light_name] = Metrics.new }
|
|
23
|
+
|
|
24
|
+
super # MonitorMixin
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @return [Array<String>]
|
|
28
|
+
def names
|
|
29
|
+
synchronize { @metrics.keys | @states.keys }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param config [Stoplight::Domain::Config]
|
|
33
|
+
# @return [Stoplight::Domain::Metrics]
|
|
34
|
+
def get_metrics(config)
|
|
35
|
+
light_name = config.name
|
|
36
|
+
|
|
37
|
+
synchronize do
|
|
38
|
+
current_time = self.current_time
|
|
39
|
+
window_start = if config.window_size
|
|
40
|
+
(current_time - config.window_size)
|
|
41
|
+
else
|
|
42
|
+
current_time
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
metrics = @metrics[light_name]
|
|
46
|
+
|
|
47
|
+
errors = @errors[light_name].sum_in_window(window_start) if config.window_size
|
|
48
|
+
successes = @successes[light_name].sum_in_window(window_start) if config.window_size
|
|
49
|
+
|
|
50
|
+
Domain::Metrics.new(
|
|
51
|
+
errors:,
|
|
52
|
+
successes:,
|
|
53
|
+
total_consecutive_errors: metrics.consecutive_errors,
|
|
54
|
+
total_consecutive_successes: metrics.consecutive_successes,
|
|
55
|
+
last_error: metrics.last_error,
|
|
56
|
+
last_success_at: metrics.last_success_at
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# @return [Stoplight::Domain::Metrics]
|
|
62
|
+
def get_recovery_metrics(config)
|
|
63
|
+
light_name = config.name
|
|
64
|
+
|
|
65
|
+
synchronize do
|
|
66
|
+
current_time = self.current_time
|
|
67
|
+
recovery_window_start = (current_time - config.cool_off_time)
|
|
68
|
+
if config.window_size
|
|
69
|
+
(current_time - config.window_size)
|
|
70
|
+
else
|
|
71
|
+
current_time
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
metrics = @metrics[light_name]
|
|
75
|
+
|
|
76
|
+
Domain::Metrics.new(
|
|
77
|
+
errors: @recovery_probe_errors[light_name].sum_in_window(recovery_window_start),
|
|
78
|
+
successes: @recovery_probe_successes[light_name].sum_in_window(recovery_window_start),
|
|
79
|
+
total_consecutive_errors: metrics.consecutive_errors,
|
|
80
|
+
total_consecutive_successes: metrics.consecutive_successes,
|
|
81
|
+
last_error: metrics.last_error,
|
|
82
|
+
last_success_at: metrics.last_success_at
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# @return [Stoplight::Domain::StateSnapshot]
|
|
88
|
+
def get_state_snapshot(config)
|
|
89
|
+
time, state = synchronize do
|
|
90
|
+
[current_time, @states[config.name]]
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
Domain::StateSnapshot.new(
|
|
94
|
+
time:,
|
|
95
|
+
locked_state: state.locked_state,
|
|
96
|
+
recovery_scheduled_after: state.recovery_scheduled_after,
|
|
97
|
+
recovery_started_at: state.recovery_started_at,
|
|
98
|
+
breached_at: state.breached_at
|
|
99
|
+
)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# @param config [Stoplight::Domain::Config]
|
|
103
|
+
# @param exception [Exception]
|
|
104
|
+
# @return [void]
|
|
105
|
+
def record_failure(config, exception)
|
|
106
|
+
current_time = self.current_time
|
|
107
|
+
light_name = config.name
|
|
108
|
+
failure = Domain::Failure.from_error(exception, time: current_time)
|
|
109
|
+
|
|
110
|
+
synchronize do
|
|
111
|
+
@errors[light_name].increment if config.window_size
|
|
112
|
+
|
|
113
|
+
metrics = @metrics[light_name]
|
|
114
|
+
|
|
115
|
+
if metrics.last_error_at.nil? || failure.occurred_at > metrics.last_error_at
|
|
116
|
+
metrics.last_error = failure
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
metrics.consecutive_errors += 1
|
|
120
|
+
metrics.consecutive_successes = 0
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def clear_windowed_metrics(config)
|
|
125
|
+
if config.window_size
|
|
126
|
+
synchronize do
|
|
127
|
+
@errors[config.name] = SlidingWindow.new
|
|
128
|
+
@successes[config.name] = SlidingWindow.new
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @param config [Stoplight::Domain::Config]
|
|
134
|
+
# @return [void]
|
|
135
|
+
def record_success(config)
|
|
136
|
+
light_name = config.name
|
|
137
|
+
current_time = self.current_time
|
|
138
|
+
|
|
139
|
+
synchronize do
|
|
140
|
+
@successes[light_name].increment if config.window_size
|
|
141
|
+
|
|
142
|
+
metrics = @metrics[light_name]
|
|
143
|
+
|
|
144
|
+
if metrics.last_success_at.nil? || current_time > metrics.last_success_at
|
|
145
|
+
metrics.last_success_at = current_time
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
metrics.consecutive_errors = 0
|
|
149
|
+
metrics.consecutive_successes += 1
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# @param config [Stoplight::Domain::Config]
|
|
154
|
+
# @param exception [Exception]
|
|
155
|
+
# @return [void]
|
|
156
|
+
def record_recovery_probe_failure(config, exception)
|
|
157
|
+
light_name = config.name
|
|
158
|
+
current_time = self.current_time
|
|
159
|
+
failure = Domain::Failure.from_error(exception, time: current_time)
|
|
160
|
+
|
|
161
|
+
synchronize do
|
|
162
|
+
@recovery_probe_errors[light_name].increment
|
|
163
|
+
|
|
164
|
+
metrics = @metrics[light_name]
|
|
165
|
+
|
|
166
|
+
if metrics.last_error_at.nil? || failure.occurred_at > metrics.last_error_at
|
|
167
|
+
metrics.last_error = failure
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
metrics.consecutive_errors += 1
|
|
171
|
+
metrics.consecutive_successes = 0
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# @param config [Stoplight::Domain::Config]
|
|
176
|
+
# @return [void]
|
|
177
|
+
def record_recovery_probe_success(config)
|
|
178
|
+
light_name = config.name
|
|
179
|
+
current_time = self.current_time
|
|
180
|
+
|
|
181
|
+
synchronize do
|
|
182
|
+
@recovery_probe_successes[light_name].increment
|
|
183
|
+
|
|
184
|
+
metrics = @metrics[light_name]
|
|
185
|
+
if metrics.last_success_at.nil? || current_time > metrics.last_success_at
|
|
186
|
+
metrics.last_success_at = current_time
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
metrics.consecutive_errors = 0
|
|
190
|
+
metrics.consecutive_successes += 1
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# @param config [Stoplight::Domain::Config]
|
|
195
|
+
# @param state [String]
|
|
196
|
+
# @return [String]
|
|
197
|
+
def set_state(config, state)
|
|
198
|
+
light_name = config.name
|
|
199
|
+
|
|
200
|
+
synchronize do
|
|
201
|
+
@states[light_name].locked_state = state
|
|
202
|
+
end
|
|
203
|
+
state
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# @return [String]
|
|
207
|
+
def inspect
|
|
208
|
+
"#<#{self.class.name}>"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Combined method that performs the state transition based on color
|
|
212
|
+
#
|
|
213
|
+
# @param config [Stoplight::Domain::Config] The light configuration
|
|
214
|
+
# @param color [String] The color to transition to ("GREEN", "YELLOW", or "RED")
|
|
215
|
+
# @return [Boolean] true if this is the first instance to detect this transition
|
|
216
|
+
def transition_to_color(config, color)
|
|
217
|
+
case color
|
|
218
|
+
when Domain::Color::GREEN
|
|
219
|
+
transition_to_green(config)
|
|
220
|
+
when Domain::Color::YELLOW
|
|
221
|
+
transition_to_yellow(config)
|
|
222
|
+
when Domain::Color::RED
|
|
223
|
+
transition_to_red(config)
|
|
224
|
+
else
|
|
225
|
+
raise ArgumentError, "Invalid color: #{color}"
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Transitions to GREEN state and ensures only one notification
|
|
230
|
+
#
|
|
231
|
+
# @param config [Stoplight::Domain::Config] The light configuration
|
|
232
|
+
# @return [Boolean] true if this is the first instance to detect this transition
|
|
233
|
+
private def transition_to_green(config)
|
|
234
|
+
light_name = config.name
|
|
235
|
+
current_time = self.current_time
|
|
236
|
+
|
|
237
|
+
synchronize do
|
|
238
|
+
state = @states[light_name]
|
|
239
|
+
|
|
240
|
+
if state.recovered_at
|
|
241
|
+
false
|
|
242
|
+
else
|
|
243
|
+
state.recovered_at = current_time
|
|
244
|
+
state.recovery_started_at = nil
|
|
245
|
+
state.breached_at = nil
|
|
246
|
+
state.recovery_scheduled_after = nil
|
|
247
|
+
true
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Transitions to YELLOW (recovery) state and ensures only one notification
|
|
253
|
+
#
|
|
254
|
+
# @param config [Stoplight::Domain::Config] The light configuration
|
|
255
|
+
# @return [Boolean] true if this is the first instance to detect this transition
|
|
256
|
+
private def transition_to_yellow(config)
|
|
257
|
+
light_name = config.name
|
|
258
|
+
current_time = self.current_time
|
|
259
|
+
|
|
260
|
+
synchronize do
|
|
261
|
+
state = @states[light_name]
|
|
262
|
+
if state.recovery_started_at.nil?
|
|
263
|
+
state.recovery_started_at = current_time
|
|
264
|
+
state.recovery_scheduled_after = nil
|
|
265
|
+
state.recovered_at = nil
|
|
266
|
+
state.breached_at = nil
|
|
267
|
+
true
|
|
268
|
+
else
|
|
269
|
+
state.recovery_scheduled_after = nil
|
|
270
|
+
state.recovered_at = nil
|
|
271
|
+
state.breached_at = nil
|
|
272
|
+
false
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Transitions to RED state and ensures only one notification
|
|
278
|
+
#
|
|
279
|
+
# @param config [Stoplight::Domain::Config] The light configuration
|
|
280
|
+
# @return [Boolean] true if this is the first instance to detect this transition
|
|
281
|
+
private def transition_to_red(config)
|
|
282
|
+
light_name = config.name
|
|
283
|
+
current_time = self.current_time
|
|
284
|
+
recovery_scheduled_after = current_time + config.cool_off_time
|
|
285
|
+
|
|
286
|
+
synchronize do
|
|
287
|
+
state = @states[light_name]
|
|
288
|
+
if state.breached_at
|
|
289
|
+
state.recovery_scheduled_after = recovery_scheduled_after
|
|
290
|
+
state.recovery_started_at = nil
|
|
291
|
+
state.recovered_at = nil
|
|
292
|
+
false
|
|
293
|
+
else
|
|
294
|
+
state.breached_at = current_time
|
|
295
|
+
state.recovery_scheduled_after = recovery_scheduled_after
|
|
296
|
+
state.recovery_started_at = nil
|
|
297
|
+
state.recovered_at = nil
|
|
298
|
+
true
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
private def current_time
|
|
304
|
+
Time.now
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
local number_of_metric_buckets = tonumber(ARGV[1])
|
|
2
|
+
local window_start_ts = tonumber(ARGV[2])
|
|
3
|
+
local window_end_ts = tonumber(ARGV[3])
|
|
4
|
+
local metrics_keys = {}
|
|
5
|
+
for idx = 4, #ARGV do
|
|
6
|
+
table.insert(metrics_keys, ARGV[idx])
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
local metadata_key = KEYS[1]
|
|
10
|
+
|
|
11
|
+
local function count_events(start_idx, bucket_count, start_ts)
|
|
12
|
+
local total = 0
|
|
13
|
+
for idx = start_idx, start_idx + bucket_count - 1 do
|
|
14
|
+
total = total + tonumber(redis.call('ZCOUNT', KEYS[idx], start_ts, window_end_ts))
|
|
15
|
+
end
|
|
16
|
+
return total
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
local offset = 2
|
|
20
|
+
local successes = count_events(2, number_of_metric_buckets, window_start_ts)
|
|
21
|
+
|
|
22
|
+
offset = offset + number_of_metric_buckets
|
|
23
|
+
local errors = count_events(offset, number_of_metric_buckets, window_start_ts)
|
|
24
|
+
|
|
25
|
+
local metrics = redis.call('HMGET', metadata_key, unpack(metrics_keys))
|
|
26
|
+
return {successes, errors, unpack(metrics)}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Stoplight
|
|
4
|
+
module Infrastructure
|
|
5
|
+
module DataStore
|
|
6
|
+
class Redis
|
|
7
|
+
# @api private
|
|
8
|
+
module Lua
|
|
9
|
+
class << self
|
|
10
|
+
def read_lua_file(name_without_extension)
|
|
11
|
+
File.read(File.join(__dir__, "#{name_without_extension}.lua"))
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
RECORD_FAILURE = read_lua_file("record_failure")
|
|
16
|
+
RECORD_SUCCESS = read_lua_file("record_success")
|
|
17
|
+
GET_METRICS = read_lua_file("get_metrics")
|
|
18
|
+
TRANSITION_TO_YELLOW = read_lua_file("transition_to_yellow")
|
|
19
|
+
TRANSITION_TO_RED = read_lua_file("transition_to_red")
|
|
20
|
+
TRANSITION_TO_GREEN = read_lua_file("transition_to_green")
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|