circuitbox 1.1.1 → 2.0.0.pre4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +53 -187
- data/lib/circuitbox/circuit_breaker/logger_messages.rb +31 -0
- data/lib/circuitbox/circuit_breaker.rb +134 -154
- data/lib/circuitbox/configuration.rb +51 -0
- data/lib/circuitbox/errors/error.rb +3 -2
- data/lib/circuitbox/errors/open_circuit_error.rb +3 -1
- data/lib/circuitbox/errors/service_failure_error.rb +5 -1
- data/lib/circuitbox/excon_middleware.rb +23 -30
- data/lib/circuitbox/faraday_middleware.rb +43 -63
- data/lib/circuitbox/memory_store/container.rb +30 -0
- data/lib/circuitbox/memory_store/monotonic_time.rb +13 -0
- data/lib/circuitbox/memory_store.rb +85 -0
- data/lib/circuitbox/notifier/active_support.rb +19 -0
- data/lib/circuitbox/notifier/null.rb +13 -0
- data/lib/circuitbox/timer.rb +51 -0
- data/lib/circuitbox/version.rb +3 -1
- data/lib/circuitbox.rb +14 -54
- metadata +106 -117
- data/.gitignore +0 -20
- data/.ruby-version +0 -1
- data/.travis.yml +0 -9
- data/Gemfile +0 -6
- data/Rakefile +0 -30
- data/benchmark/circuit_store_benchmark.rb +0 -114
- data/circuitbox.gemspec +0 -48
- data/lib/circuitbox/notifier.rb +0 -34
- data/test/circuit_breaker_test.rb +0 -436
- data/test/circuitbox_test.rb +0 -45
- data/test/excon_middleware_test.rb +0 -131
- data/test/faraday_middleware_test.rb +0 -175
- data/test/integration/circuitbox_cross_process_open_test.rb +0 -56
- data/test/integration/faraday_middleware_test.rb +0 -78
- data/test/integration_helper.rb +0 -48
- data/test/notifier_test.rb +0 -21
- data/test/service_failure_error_test.rb +0 -23
- data/test/test_helper.rb +0 -15
@@ -1,15 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'circuit_breaker/logger_messages'
|
4
|
+
|
1
5
|
class Circuitbox
|
2
6
|
class CircuitBreaker
|
3
|
-
|
4
|
-
|
7
|
+
include LoggerMessages
|
8
|
+
|
9
|
+
attr_reader :service, :circuit_options, :exceptions,
|
10
|
+
:logger, :circuit_store, :notifier, :time_class
|
5
11
|
|
6
12
|
DEFAULTS = {
|
7
|
-
sleep_window:
|
13
|
+
sleep_window: 90,
|
8
14
|
volume_threshold: 5,
|
9
|
-
error_threshold:
|
10
|
-
|
11
|
-
|
12
|
-
}
|
15
|
+
error_threshold: 50,
|
16
|
+
time_window: 60
|
17
|
+
}.freeze
|
13
18
|
|
14
19
|
#
|
15
20
|
# Configuration options
|
@@ -17,238 +22,213 @@ class Circuitbox
|
|
17
22
|
# `sleep_window` - seconds to sleep the circuit
|
18
23
|
# `volume_threshold` - number of requests before error rate calculation occurs
|
19
24
|
# `error_threshold` - percentage of failed requests needed to trip circuit
|
20
|
-
# `
|
21
|
-
# `exceptions` - exceptions other than Timeout::Error that count as failures
|
25
|
+
# `exceptions` - exceptions that count as failures
|
22
26
|
# `time_window` - interval of time used to calculate error_rate (in seconds) - default is 60s
|
23
27
|
# `logger` - Logger to use - defaults to Rails.logger if defined, otherwise STDOUT
|
24
28
|
#
|
25
29
|
def initialize(service, options = {})
|
26
|
-
@service = service
|
27
|
-
@circuit_options = options
|
28
|
-
@circuit_store = options.fetch(:cache) { Circuitbox.
|
29
|
-
@notifier
|
30
|
+
@service = service.to_s
|
31
|
+
@circuit_options = DEFAULTS.merge(options)
|
32
|
+
@circuit_store = options.fetch(:cache) { Circuitbox.default_circuit_store }
|
33
|
+
@notifier = options.fetch(:notifier) { Circuitbox.default_notifier }
|
34
|
+
|
35
|
+
if @circuit_options[:timeout_seconds]
|
36
|
+
warn('timeout_seconds was removed in circuitbox 2.0. '\
|
37
|
+
'Check the upgrade guide at https://github.com/yammer/circuitbox')
|
38
|
+
end
|
30
39
|
|
31
|
-
@exceptions = options.fetch(:exceptions)
|
32
|
-
|
40
|
+
@exceptions = options.fetch(:exceptions)
|
41
|
+
raise ArgumentError.new('exceptions need to be an array') unless @exceptions.is_a?(Array)
|
33
42
|
|
34
|
-
@logger = options.fetch(:logger) {
|
35
|
-
@time_class
|
36
|
-
|
43
|
+
@logger = options.fetch(:logger) { Circuitbox.default_logger }
|
44
|
+
@time_class = options.fetch(:time_class) { Time }
|
45
|
+
@state_change_mutex = Mutex.new
|
46
|
+
check_sleep_window
|
37
47
|
end
|
38
48
|
|
39
49
|
def option_value(name)
|
40
|
-
value = circuit_options
|
50
|
+
value = circuit_options[name]
|
41
51
|
value.is_a?(Proc) ? value.call : value
|
42
52
|
end
|
43
53
|
|
44
|
-
def run
|
45
|
-
@partition = run_options.delete(:partition) # sorry for this hack.
|
46
|
-
|
54
|
+
def run(circuitbox_exceptions: true, &block)
|
47
55
|
if open?
|
48
|
-
logger.debug "[CIRCUIT] open: skipping #{service}"
|
49
|
-
open! unless open_flag?
|
50
56
|
skipped!
|
51
|
-
raise Circuitbox::OpenCircuitError.new(service)
|
57
|
+
raise Circuitbox::OpenCircuitError.new(service) if circuitbox_exceptions
|
52
58
|
else
|
53
|
-
|
54
|
-
logger.debug "[CIRCUIT] closed: querying #{service}"
|
59
|
+
logger.debug(circuit_running_message)
|
55
60
|
|
56
61
|
begin
|
57
|
-
response =
|
58
|
-
|
59
|
-
timeout (timeout_seconds) { yield }
|
60
|
-
else
|
61
|
-
yield
|
62
|
-
end
|
63
|
-
|
64
|
-
logger.debug "[CIRCUIT] closed: #{service} querie success"
|
62
|
+
response = Timer.measure(service, notifier, 'runtime', &block)
|
63
|
+
|
65
64
|
success!
|
66
|
-
rescue *exceptions =>
|
67
|
-
|
65
|
+
rescue *exceptions => e
|
66
|
+
# Other stores could raise an exception that circuitbox is asked to watch.
|
67
|
+
# setting to nil keeps the same behavior as the previous defination of run.
|
68
|
+
response = nil
|
68
69
|
failure!
|
69
|
-
|
70
|
-
raise Circuitbox::ServiceFailureError.new(service, exception)
|
70
|
+
raise Circuitbox::ServiceFailureError.new(service, e) if circuitbox_exceptions
|
71
71
|
end
|
72
72
|
end
|
73
73
|
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
def run(run_options = {})
|
78
|
-
begin
|
79
|
-
run!(run_options, &Proc.new)
|
80
|
-
rescue Circuitbox::Error
|
81
|
-
nil
|
82
|
-
end
|
74
|
+
response
|
83
75
|
end
|
84
76
|
|
85
77
|
def open?
|
86
|
-
|
87
|
-
true
|
88
|
-
elsif passed_volume_threshold? && passed_rate_threshold?
|
89
|
-
true
|
90
|
-
else
|
91
|
-
false
|
92
|
-
end
|
78
|
+
circuit_store.key?(open_storage_key)
|
93
79
|
end
|
94
80
|
|
95
81
|
def error_rate(failures = failure_count, success = success_count)
|
96
82
|
all_count = failures + success
|
97
|
-
return 0.0 unless all_count
|
98
|
-
|
83
|
+
return 0.0 unless all_count.positive?
|
84
|
+
|
85
|
+
(failures / all_count.to_f) * 100
|
99
86
|
end
|
100
87
|
|
101
88
|
def failure_count
|
102
|
-
circuit_store.load(stat_storage_key(
|
89
|
+
circuit_store.load(stat_storage_key('failure'), raw: true).to_i
|
103
90
|
end
|
104
91
|
|
105
92
|
def success_count
|
106
|
-
circuit_store.load(stat_storage_key(
|
93
|
+
circuit_store.load(stat_storage_key('success'), raw: true).to_i
|
107
94
|
end
|
108
95
|
|
109
96
|
def try_close_next_time
|
110
|
-
circuit_store.delete(
|
97
|
+
circuit_store.delete(open_storage_key)
|
111
98
|
end
|
112
99
|
|
113
|
-
|
114
|
-
def open!
|
115
|
-
log_event :open
|
116
|
-
logger.debug "[CIRCUIT] opening #{service} circuit"
|
117
|
-
circuit_store.store(storage_key(:asleep), true, expires: option_value(:sleep_window))
|
118
|
-
half_open!
|
119
|
-
was_open!
|
120
|
-
end
|
100
|
+
private
|
121
101
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
102
|
+
def should_open?
|
103
|
+
failures = failure_count
|
104
|
+
successes = success_count
|
105
|
+
rate = error_rate(failures, successes)
|
106
|
+
|
107
|
+
passed_volume_threshold?(failures, successes) && passed_rate_threshold?(rate)
|
126
108
|
end
|
127
109
|
|
128
|
-
def
|
129
|
-
|
110
|
+
def passed_volume_threshold?(failures, successes)
|
111
|
+
failures + successes >= option_value(:volume_threshold)
|
130
112
|
end
|
131
113
|
|
132
|
-
def
|
133
|
-
|
114
|
+
def passed_rate_threshold?(rate)
|
115
|
+
rate >= option_value(:error_threshold)
|
134
116
|
end
|
135
|
-
### END
|
136
117
|
|
137
|
-
def
|
138
|
-
|
118
|
+
def half_open_failure
|
119
|
+
@state_change_mutex.synchronize do
|
120
|
+
return if open? || !half_open?
|
121
|
+
|
122
|
+
trip
|
123
|
+
end
|
124
|
+
|
125
|
+
# Running event and logger outside of the synchronize block to allow other threads
|
126
|
+
# that may be waiting to become unblocked
|
127
|
+
notify_opened
|
139
128
|
end
|
140
129
|
|
141
|
-
def
|
142
|
-
|
130
|
+
def open!
|
131
|
+
@state_change_mutex.synchronize do
|
132
|
+
return if open?
|
133
|
+
|
134
|
+
trip
|
135
|
+
end
|
136
|
+
|
137
|
+
# Running event and logger outside of the synchronize block to allow other threads
|
138
|
+
# that may be waiting to become unblocked
|
139
|
+
notify_opened
|
143
140
|
end
|
144
141
|
|
145
|
-
def
|
146
|
-
|
142
|
+
def notify_opened
|
143
|
+
notify_event('open')
|
144
|
+
logger.debug(circuit_opened_message)
|
147
145
|
end
|
148
146
|
|
149
|
-
def
|
150
|
-
|
147
|
+
def trip
|
148
|
+
circuit_store.store(open_storage_key, true, expires: option_value(:sleep_window))
|
149
|
+
circuit_store.store(half_open_storage_key, true)
|
151
150
|
end
|
152
151
|
|
153
|
-
def
|
154
|
-
|
152
|
+
def close!
|
153
|
+
@state_change_mutex.synchronize do
|
154
|
+
# If the circuit is not open, the half_open key will be deleted from the store
|
155
|
+
# if half_open exists the deleted value is returned and allows us to continue
|
156
|
+
# if half_open doesn't exist nil is returned, causing us to return early
|
157
|
+
return unless !open? && circuit_store.delete(half_open_storage_key)
|
158
|
+
end
|
159
|
+
|
160
|
+
# Running event outside of the synchronize block to allow other threads
|
161
|
+
# that may be waiting to become unblocked
|
162
|
+
notify_event('close')
|
163
|
+
logger.debug(circuit_closed_message)
|
155
164
|
end
|
156
165
|
|
157
|
-
def
|
158
|
-
|
159
|
-
success = success_count
|
160
|
-
rate = error_rate(failures, success)
|
161
|
-
log_metrics(rate, failures, success)
|
162
|
-
rate
|
166
|
+
def half_open?
|
167
|
+
circuit_store.key?(half_open_storage_key)
|
163
168
|
end
|
164
169
|
|
165
170
|
def success!
|
166
|
-
|
167
|
-
|
171
|
+
increment_and_notify_event('success')
|
172
|
+
logger.debug(circuit_success_message)
|
173
|
+
|
174
|
+
close! if half_open?
|
168
175
|
end
|
169
176
|
|
170
177
|
def failure!
|
171
|
-
|
178
|
+
increment_and_notify_event('failure')
|
179
|
+
logger.debug(circuit_failure_message)
|
180
|
+
|
181
|
+
if half_open?
|
182
|
+
half_open_failure
|
183
|
+
elsif should_open?
|
184
|
+
open!
|
185
|
+
end
|
172
186
|
end
|
173
187
|
|
174
188
|
def skipped!
|
175
|
-
|
189
|
+
notify_event('skipped')
|
190
|
+
logger.debug(circuit_skipped_message)
|
176
191
|
end
|
177
192
|
|
178
|
-
#
|
179
|
-
def
|
180
|
-
notifier.
|
181
|
-
log_event_to_process(event)
|
193
|
+
# Send event notification to notifier
|
194
|
+
def notify_event(event)
|
195
|
+
notifier.notify(service, event)
|
182
196
|
end
|
183
197
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
n.metric_gauge(:success_count, successes)
|
198
|
+
# Increment stat store and send notification
|
199
|
+
def increment_and_notify_event(event)
|
200
|
+
circuit_store.increment(stat_storage_key(event), 1, expires: (option_value(:time_window) * 2))
|
201
|
+
notify_event(event)
|
189
202
|
end
|
190
203
|
|
191
|
-
def
|
204
|
+
def check_sleep_window
|
192
205
|
sleep_window = option_value(:sleep_window)
|
193
206
|
time_window = option_value(:time_window)
|
194
|
-
|
195
|
-
notifier.new(service,partition).notify_warning("sleep_window:#{sleep_window} is shorter than time_window:#{time_window}, the error_rate could not be reset properly after a sleep. sleep_window as been set to equal time_window.")
|
196
|
-
@circuit_options[:sleep_window] = option_value(:time_window)
|
197
|
-
end
|
198
|
-
end
|
199
|
-
|
200
|
-
# When there is a successful response within a count interval, clear the failures.
|
201
|
-
def clear_failures!
|
202
|
-
circuit_store.store(stat_storage_key(:failure), 0, raw: true)
|
203
|
-
end
|
207
|
+
return unless sleep_window < time_window
|
204
208
|
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
circuit_store.increment(key)
|
210
|
-
else
|
211
|
-
# yes we want a string here, as the underlying stores impement this as a native type.
|
212
|
-
circuit_store.store(key, "1", raw: true)
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
# For returning stale responses when the circuit is open
|
217
|
-
def response_key(args)
|
218
|
-
Digest::SHA1.hexdigest(storage_key(:cache, args.inspect.to_s))
|
209
|
+
warning_message = "sleep_window: #{sleep_window} is shorter than time_window: #{time_window}, "\
|
210
|
+
"the error_rate would not be reset after a sleep."
|
211
|
+
notifier.notify_warning(service, warning_message)
|
212
|
+
warn("Circuit: #{service}, Warning: #{warning_message}")
|
219
213
|
end
|
220
214
|
|
221
|
-
def stat_storage_key(event
|
222
|
-
|
215
|
+
def stat_storage_key(event)
|
216
|
+
"circuits:#{service}:stats:#{align_time_to_window}:#{event}"
|
223
217
|
end
|
224
218
|
|
225
|
-
|
226
219
|
# return time representation in seconds
|
227
|
-
def
|
228
|
-
time
|
220
|
+
def align_time_to_window
|
221
|
+
time = time_class.now.to_i
|
229
222
|
time_window = option_value(:time_window)
|
230
|
-
time - (
|
231
|
-
end
|
232
|
-
|
233
|
-
def storage_key(*args)
|
234
|
-
options = args.extract_options!
|
235
|
-
|
236
|
-
key = if options[:without_partition]
|
237
|
-
"circuits:#{service}:#{args.join(":")}"
|
238
|
-
else
|
239
|
-
"circuits:#{service}:#{partition}:#{args.join(":")}"
|
240
|
-
end
|
241
|
-
|
242
|
-
return key
|
223
|
+
time - (time % time_window) # remove rest of integer division
|
243
224
|
end
|
244
225
|
|
245
|
-
def
|
246
|
-
|
226
|
+
def open_storage_key
|
227
|
+
@open_storage_key ||= "circuits:#{service}:open"
|
247
228
|
end
|
248
229
|
|
249
|
-
def
|
250
|
-
|
230
|
+
def half_open_storage_key
|
231
|
+
@half_open_storage_key ||= "circuits:#{service}:half_open"
|
251
232
|
end
|
252
|
-
|
253
233
|
end
|
254
234
|
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'memory_store'
|
4
|
+
require_relative 'timer'
|
5
|
+
require_relative 'notifier/active_support'
|
6
|
+
require_relative 'notifier/null'
|
7
|
+
|
8
|
+
class Circuitbox
|
9
|
+
module Configuration
|
10
|
+
attr_writer :default_circuit_store,
|
11
|
+
:default_notifier,
|
12
|
+
:default_timer,
|
13
|
+
:default_logger
|
14
|
+
|
15
|
+
def configure
|
16
|
+
yield self
|
17
|
+
clear_cached_circuits!
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def default_circuit_store
|
22
|
+
@default_circuit_store ||= MemoryStore.new
|
23
|
+
end
|
24
|
+
|
25
|
+
def default_notifier
|
26
|
+
@default_notifier ||= if defined?(ActiveSupport::Notifications)
|
27
|
+
Notifier::ActiveSupport.new
|
28
|
+
else
|
29
|
+
Notifier::Null.new
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def default_logger
|
34
|
+
@default_logger ||= if defined?(Rails)
|
35
|
+
Rails.logger
|
36
|
+
else
|
37
|
+
Logger.new($stdout)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def cached_circuits
|
44
|
+
@cached_circuits ||= {}
|
45
|
+
end
|
46
|
+
|
47
|
+
def clear_cached_circuits!
|
48
|
+
@cached_circuits = {}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -1,12 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class Circuitbox
|
2
4
|
class ServiceFailureError < Circuitbox::Error
|
3
5
|
attr_reader :service, :original
|
4
6
|
|
5
7
|
def initialize(service, exception)
|
8
|
+
super()
|
6
9
|
@service = service
|
7
10
|
@original = exception
|
8
11
|
# we copy over the original exceptions backtrace if there is one
|
9
|
-
|
12
|
+
backtrace = exception.backtrace
|
13
|
+
set_backtrace(backtrace) unless backtrace.empty?
|
10
14
|
end
|
11
15
|
|
12
16
|
def to_s
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'excon'
|
2
4
|
require 'circuitbox'
|
3
5
|
|
@@ -8,7 +10,7 @@ class Circuitbox
|
|
8
10
|
DEFAULT_EXCEPTIONS = [
|
9
11
|
Excon::Errors::Timeout,
|
10
12
|
RequestFailed
|
11
|
-
]
|
13
|
+
].freeze
|
12
14
|
|
13
15
|
class NullResponse < Excon::Response
|
14
16
|
def initialize(response, exception)
|
@@ -26,36 +28,36 @@ class Circuitbox
|
|
26
28
|
|
27
29
|
def initialize(stack, opts = {})
|
28
30
|
@stack = stack
|
29
|
-
default_options = { open_circuit:
|
31
|
+
default_options = { open_circuit: ->(response) { response[:status] >= 400 } }
|
30
32
|
@opts = default_options.merge(opts)
|
31
33
|
super(stack)
|
32
34
|
end
|
33
35
|
|
34
36
|
def error_call(datum)
|
35
|
-
circuit(datum).run
|
37
|
+
circuit(datum).run do
|
36
38
|
raise RequestFailed
|
37
39
|
end
|
38
|
-
rescue Circuitbox::Error =>
|
39
|
-
circuit_open_value(datum, datum[:response],
|
40
|
+
rescue Circuitbox::Error => e
|
41
|
+
circuit_open_value(datum, datum[:response], e)
|
40
42
|
end
|
41
43
|
|
42
44
|
def request_call(datum)
|
43
|
-
circuit(datum).run
|
45
|
+
circuit(datum).run do
|
44
46
|
@stack.request_call(datum)
|
45
47
|
end
|
46
48
|
end
|
47
49
|
|
48
50
|
def response_call(datum)
|
49
|
-
circuit(datum).run
|
51
|
+
circuit(datum).run do
|
50
52
|
raise RequestFailed if open_circuit?(datum[:response])
|
51
53
|
end
|
52
54
|
@stack.response_call(datum)
|
53
|
-
rescue Circuitbox::Error =>
|
54
|
-
circuit_open_value(datum, datum[:response],
|
55
|
+
rescue Circuitbox::Error => e
|
56
|
+
circuit_open_value(datum, datum[:response], e)
|
55
57
|
end
|
56
58
|
|
57
59
|
def identifier
|
58
|
-
@identifier ||= opts.fetch(:identifier, ->(env) { env[:
|
60
|
+
@identifier ||= opts.fetch(:identifier, ->(env) { env[:host] })
|
59
61
|
end
|
60
62
|
|
61
63
|
def exceptions
|
@@ -69,10 +71,6 @@ class Circuitbox
|
|
69
71
|
circuitbox.circuit id, circuit_breaker_options
|
70
72
|
end
|
71
73
|
|
72
|
-
def run_options(datum)
|
73
|
-
opts.merge(datum)[:circuit_breaker_run_options] || {}
|
74
|
-
end
|
75
|
-
|
76
74
|
def open_circuit?(response)
|
77
75
|
opts[:open_circuit].call(response)
|
78
76
|
end
|
@@ -86,26 +84,21 @@ class Circuitbox
|
|
86
84
|
end
|
87
85
|
|
88
86
|
def circuit_breaker_options
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
87
|
+
@circuit_breaker_options ||= begin
|
88
|
+
options = opts.fetch(:circuit_breaker_options, {})
|
89
|
+
options.merge!(
|
90
|
+
exceptions: opts.fetch(:exceptions, DEFAULT_EXCEPTIONS)
|
91
|
+
)
|
92
|
+
end
|
95
93
|
end
|
96
94
|
|
97
95
|
def default_value
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
96
|
+
@default_value ||= begin
|
97
|
+
default = opts.fetch(:default_value) do
|
98
|
+
->(response, exception) { NullResponse.new(response, exception) }
|
99
|
+
end
|
100
|
+
default.respond_to?(:call) ? default : ->(*) { default }
|
102
101
|
end
|
103
|
-
|
104
|
-
@default_value = if default.respond_to?(:call)
|
105
|
-
default
|
106
|
-
else
|
107
|
-
lambda { |*| default }
|
108
|
-
end
|
109
102
|
end
|
110
103
|
end
|
111
104
|
end
|