canvas_sync 0.17.3.beta3 → 0.17.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/job_batches/batch.rb +40 -13
- data/lib/canvas_sync/job_batches/callback.rb +15 -16
- data/lib/canvas_sync/job_batches/context_hash.rb +6 -1
- data/lib/canvas_sync/job_batches/pool.rb +38 -18
- data/lib/canvas_sync/job_batches/sidekiq.rb +2 -0
- data/lib/canvas_sync/job_batches/sidekiq/web.rb +114 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +41 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb +42 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb +26 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb +138 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb +23 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb +85 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb +47 -0
- data/lib/canvas_sync/job_batches/status.rb +9 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/spec/dummy/log/test.log +57826 -0
- data/spec/job_batching/batch_spec.rb +0 -1
- data/spec/job_batching/context_hash_spec.rb +54 -0
- data/spec/job_batching/status_spec.rb +3 -3
- data/spec/spec_helper.rb +3 -7
- metadata +14 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 999dfad29e32b30e204cc26e6f36700752f3f3efc06314293c9d86df1b52b774
|
4
|
+
data.tar.gz: ef2c1b967f0451a3f2029d2aaae5dbe698e93a5a0136a52eeefe184dc1009f36
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd96c186dced2cb00a77fe8e729094ce8c9a7878f74c854c6e15fe1e143b773682b747065c231cf9babf777ebe752f9bdb24f130b7b2485dab9efae7e4ba1be9
|
7
|
+
data.tar.gz: c7d3f33ccbc78a5f4fc22ab1c834ebf21c2be0863e7ff6cbabb5af5ebb5f9181783d8a18f2a1a015d56193d0a67289e1f77f12dcd5942d0b42e3ef6e96b82317
|
@@ -99,7 +99,7 @@ module CanvasSync
|
|
99
99
|
if parent_bid
|
100
100
|
r.hincrby("BID-#{parent_bid}", "children", 1)
|
101
101
|
r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
|
102
|
-
r.
|
102
|
+
r.zadd("BID-#{parent_bid}-bids", created_at, bid)
|
103
103
|
end
|
104
104
|
end
|
105
105
|
end
|
@@ -171,6 +171,13 @@ module CanvasSync
|
|
171
171
|
@bidkey
|
172
172
|
end
|
173
173
|
|
174
|
+
def flush_pending_attrs
|
175
|
+
super
|
176
|
+
redis do |r|
|
177
|
+
r.zadd("batches", created_at, bid)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
174
181
|
private
|
175
182
|
|
176
183
|
def assert_batch_is_open
|
@@ -184,15 +191,16 @@ module CanvasSync
|
|
184
191
|
|
185
192
|
def append_jobs(jids)
|
186
193
|
jids = jids.uniq
|
194
|
+
return unless jids.size > 0
|
195
|
+
|
187
196
|
redis do |r|
|
197
|
+
tme = Time.now.utc.to_f
|
198
|
+
added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
|
188
199
|
r.multi do
|
189
|
-
r.hincrby(@bidkey, "pending",
|
200
|
+
r.hincrby(@bidkey, "pending", added)
|
201
|
+
r.hincrby(@bidkey, "job_count", added)
|
190
202
|
r.expire(@bidkey, BID_EXPIRE_TTL)
|
191
|
-
|
192
|
-
if jids.size > 0
|
193
|
-
r.sadd(@bidkey + "-jids", jids)
|
194
|
-
r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
|
195
|
-
end
|
203
|
+
r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
|
196
204
|
end
|
197
205
|
end
|
198
206
|
end
|
@@ -200,6 +208,8 @@ module CanvasSync
|
|
200
208
|
class << self
|
201
209
|
def process_failed_job(bid, jid)
|
202
210
|
_, pending, failed, children, complete, parent_bid = redis do |r|
|
211
|
+
return unless r.exists?("BID-#{bid}")
|
212
|
+
|
203
213
|
r.multi do
|
204
214
|
r.sadd("BID-#{bid}-failed", jid)
|
205
215
|
|
@@ -220,6 +230,8 @@ module CanvasSync
|
|
220
230
|
|
221
231
|
def process_dead_job(bid, jid)
|
222
232
|
_, failed, children, complete, parent_bid = redis do |r|
|
233
|
+
return unless r.exists?("BID-#{bid}")
|
234
|
+
|
223
235
|
r.multi do
|
224
236
|
r.sadd("BID-#{bid}-dead", jid)
|
225
237
|
|
@@ -246,6 +258,8 @@ module CanvasSync
|
|
246
258
|
|
247
259
|
def process_successful_job(bid, jid)
|
248
260
|
_, failed, pending, children, complete, success, parent_bid = redis do |r|
|
261
|
+
return unless r.exists?("BID-#{bid}")
|
262
|
+
|
249
263
|
r.multi do
|
250
264
|
r.srem("BID-#{bid}-failed", jid)
|
251
265
|
|
@@ -257,13 +271,11 @@ module CanvasSync
|
|
257
271
|
r.hget("BID-#{bid}", "parent_bid")
|
258
272
|
|
259
273
|
r.hincrby("BID-#{bid}", "successful-jobs", 1)
|
260
|
-
r.
|
274
|
+
r.zrem("BID-#{bid}-jids", jid)
|
261
275
|
r.expire("BID-#{bid}", BID_EXPIRE_TTL)
|
262
276
|
end
|
263
277
|
end
|
264
278
|
|
265
|
-
# TODO - There seems to be an issue where a :complete callback batch will occasionally run its job, but won't enqueue_callbacks()
|
266
|
-
|
267
279
|
all_success = pending.to_i.zero? && children == success
|
268
280
|
# if complete or successfull call complete callback (the complete callback may then call successful)
|
269
281
|
if (pending.to_i == failed.to_i && children == complete) || all_success
|
@@ -302,6 +314,11 @@ module CanvasSync
|
|
302
314
|
if callback_args.present? && !callback_params.present?
|
303
315
|
logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
|
304
316
|
|
317
|
+
redis do |r|
|
318
|
+
r.sadd("#{batch_key}-pending_callbacks", event)
|
319
|
+
r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
|
320
|
+
end
|
321
|
+
|
305
322
|
with_batch(parent_bid) do
|
306
323
|
cb_batch = self.new
|
307
324
|
cb_batch.callback_params = {
|
@@ -330,7 +347,8 @@ module CanvasSync
|
|
330
347
|
def cleanup_redis(bid)
|
331
348
|
logger.debug {"Cleaning redis of batch #{bid}"}
|
332
349
|
redis do |r|
|
333
|
-
r.
|
350
|
+
r.zrem("batches", bid)
|
351
|
+
r.unlink(
|
334
352
|
"BID-#{bid}",
|
335
353
|
"BID-#{bid}-callbacks-complete",
|
336
354
|
"BID-#{bid}-callbacks-success",
|
@@ -341,10 +359,21 @@ module CanvasSync
|
|
341
359
|
"BID-#{bid}-batches-failed",
|
342
360
|
"BID-#{bid}-bids",
|
343
361
|
"BID-#{bid}-jids",
|
362
|
+
"BID-#{bid}-pending_callbacks",
|
344
363
|
)
|
345
364
|
end
|
346
365
|
end
|
347
366
|
|
367
|
+
def delete_prematurely!(bid)
|
368
|
+
child_bids = redis do |r|
|
369
|
+
r.zrange("BID-#{bid}-bids", 0, -1)
|
370
|
+
end
|
371
|
+
child_bids.each do |cbid|
|
372
|
+
delete_prematurely!(cbid)
|
373
|
+
end
|
374
|
+
cleanup_redis(bid)
|
375
|
+
end
|
376
|
+
|
348
377
|
def redis(*args, &blk)
|
349
378
|
defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
|
350
379
|
end
|
@@ -353,8 +382,6 @@ module CanvasSync
|
|
353
382
|
defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
|
354
383
|
end
|
355
384
|
|
356
|
-
private
|
357
|
-
|
358
385
|
def push_callbacks(args, queue)
|
359
386
|
Batch::Callback::worker_class.enqueue_all(args, queue)
|
360
387
|
end
|
@@ -52,9 +52,6 @@ module CanvasSync
|
|
52
52
|
|
53
53
|
class Finalize
|
54
54
|
def dispatch(status, opts)
|
55
|
-
is_callback_batch = opts['origin'].present?
|
56
|
-
has_callback_batch = opts['callback_bid'].present?
|
57
|
-
|
58
55
|
bid = opts["bid"]
|
59
56
|
event = opts["event"].to_sym
|
60
57
|
|
@@ -63,12 +60,20 @@ module CanvasSync
|
|
63
60
|
batch_status = Status.new bid
|
64
61
|
send(event, bid, batch_status, batch_status.parent_bid)
|
65
62
|
|
66
|
-
if event == :success
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
63
|
+
if event == :success
|
64
|
+
if opts['origin'].present?
|
65
|
+
origin_bid = opts['origin']['for_bid']
|
66
|
+
_, pending, success_ran = Batch.redis do |r|
|
67
|
+
r.multi do
|
68
|
+
r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
|
69
|
+
r.scard("BID-#{origin_bid}-pending_callbacks")
|
70
|
+
r.hget("BID-#{origin_bid}", "success")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
|
74
|
+
elsif (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
|
75
|
+
Batch.cleanup_redis(bid)
|
76
|
+
end
|
72
77
|
end
|
73
78
|
end
|
74
79
|
|
@@ -113,13 +118,7 @@ module CanvasSync
|
|
113
118
|
end
|
114
119
|
end
|
115
120
|
|
116
|
-
|
117
|
-
# Also, only trigger the success callback if the :complete callback_batch was successful
|
118
|
-
if pending.to_i.zero? && children == success
|
119
|
-
# Batch.enqueue_callbacks(:success, bid)
|
120
|
-
|
121
|
-
# otherwise check for a parent and call its :complete if needed
|
122
|
-
elsif parent_bid
|
121
|
+
if parent_bid && !(pending.to_i.zero? && children == success)
|
123
122
|
# if batch was not successfull check and see if its parent is complete
|
124
123
|
# if the parent is complete we trigger the complete callback
|
125
124
|
# We don't want to run this if the batch was successfull because the success
|
@@ -28,6 +28,10 @@ module CanvasSync
|
|
28
28
|
@hash_map[local_bid]
|
29
29
|
end
|
30
30
|
|
31
|
+
def own
|
32
|
+
resolve_hash(@bid_stack[-1]) || {}
|
33
|
+
end
|
34
|
+
|
31
35
|
def set_local(new_hash)
|
32
36
|
@dirty = true
|
33
37
|
local.clear.merge!(new_hash)
|
@@ -106,7 +110,8 @@ module CanvasSync
|
|
106
110
|
@bid_stack[index]
|
107
111
|
else
|
108
112
|
pbid = Batch.redis do |r|
|
109
|
-
r.hget("BID-#{bid}", "
|
113
|
+
callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
|
114
|
+
callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
|
110
115
|
end
|
111
116
|
@bid_stack.unshift(pbid)
|
112
117
|
pbid
|
@@ -24,6 +24,7 @@ module CanvasSync
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def self.from_pid(pid)
|
27
|
+
raise "PID must be given" unless pid.present?
|
27
28
|
new(pid)
|
28
29
|
end
|
29
30
|
|
@@ -38,7 +39,7 @@ module CanvasSync
|
|
38
39
|
def add_jobs(job_descs)
|
39
40
|
job_descs.each do |job_desc|
|
40
41
|
wrapper = Batch.new
|
41
|
-
wrapper.description = "Pool Job Wrapper"
|
42
|
+
wrapper.description = "Pool Job Wrapper (PID: #{pid})"
|
42
43
|
checkin_event = (on_failed_job == :wait) ? :success : :complete
|
43
44
|
wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
|
44
45
|
wrapper.jobs {}
|
@@ -57,19 +58,42 @@ module CanvasSync
|
|
57
58
|
def cleanup_redis
|
58
59
|
Batch.logger.debug {"Cleaning redis of pool #{pid}"}
|
59
60
|
redis do |r|
|
60
|
-
r.
|
61
|
+
r.zrem("pools", pid)
|
62
|
+
r.unlink(
|
61
63
|
"#{redis_key}",
|
62
64
|
"#{redis_key}-jobs",
|
63
65
|
)
|
64
66
|
end
|
65
67
|
end
|
66
68
|
|
69
|
+
def active_count
|
70
|
+
redis do |r|
|
71
|
+
r.hincrby(redis_key, "active_count", 0)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def pending_count
|
76
|
+
jobs_key = "#{redis_key}-jobs"
|
77
|
+
order = self.order || 'fifo'
|
78
|
+
redis do |r|
|
79
|
+
case order.to_sym
|
80
|
+
when :fifo, :lifo
|
81
|
+
r.llen(jobs_key)
|
82
|
+
when :random
|
83
|
+
r.scard(jobs_key)
|
84
|
+
when :priority
|
85
|
+
r.zcard(jobs_key)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
67
90
|
def job_checked_in(status, options)
|
68
91
|
active_count = redis do |r|
|
92
|
+
return unless r.exists?(redis_key)
|
69
93
|
r.hincrby(redis_key, "active_count", -1)
|
70
94
|
end
|
71
|
-
added_count = refill_allotment
|
72
95
|
|
96
|
+
added_count = refill_allotment
|
73
97
|
if active_count == 0 && added_count == 0
|
74
98
|
cleanup_redis if clean_when_empty
|
75
99
|
end
|
@@ -159,29 +183,25 @@ module CanvasSync
|
|
159
183
|
ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
|
160
184
|
end
|
161
185
|
|
162
|
-
def pending_count
|
163
|
-
order = self.order
|
164
|
-
redis do |r|
|
165
|
-
case order.to_sym
|
166
|
-
when :fifo, :lifo
|
167
|
-
r.llen(jobs_key)
|
168
|
-
when :random
|
169
|
-
r.scard(jobs_key)
|
170
|
-
when :priority
|
171
|
-
r.zcard(jobs_key)
|
172
|
-
end
|
173
|
-
end
|
174
|
-
end
|
175
|
-
|
176
186
|
def self.redis(&blk)
|
177
187
|
Batch.redis &blk
|
178
188
|
end
|
179
189
|
delegate :redis, to: :class
|
180
190
|
|
191
|
+
protected
|
192
|
+
|
193
|
+
def flush_pending_attrs
|
194
|
+
super
|
195
|
+
redis do |r|
|
196
|
+
r.zadd("pools", created_at, pid)
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
181
200
|
private
|
182
201
|
|
183
|
-
def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
|
202
|
+
def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
|
184
203
|
self.created_at = Time.now.utc.to_f
|
204
|
+
self.description = description
|
185
205
|
self.order = order
|
186
206
|
self.concurrency = concurrency
|
187
207
|
self.clean_when_empty = clean_when_empty
|
@@ -0,0 +1,114 @@
|
|
1
|
+
|
2
|
+
begin
|
3
|
+
require "sidekiq/web"
|
4
|
+
rescue LoadError
|
5
|
+
# client-only usage
|
6
|
+
end
|
7
|
+
|
8
|
+
require_relative "web/helpers"
|
9
|
+
|
10
|
+
module CanvasSync::JobBatches::Sidekiq
|
11
|
+
module Web
|
12
|
+
def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
13
|
+
app.helpers do
|
14
|
+
include Web::Helpers
|
15
|
+
end
|
16
|
+
|
17
|
+
app.get "/batches" do
|
18
|
+
@count = (params['count'] || 25).to_i
|
19
|
+
@current_page, @total_size, @batches = page('batches', params['page'], @count)
|
20
|
+
@batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
|
21
|
+
|
22
|
+
erb(get_template(:batches))
|
23
|
+
end
|
24
|
+
|
25
|
+
app.get "/batches/:bid" do
|
26
|
+
@bid = params[:bid]
|
27
|
+
@batch = CanvasSync::JobBatches::Batch.new(@bid)
|
28
|
+
|
29
|
+
@count = (params['count'] || 25).to_i
|
30
|
+
@current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
|
31
|
+
@sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
|
32
|
+
|
33
|
+
@current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
|
34
|
+
@jobs = @jobs.map {|jid, score| jid }
|
35
|
+
|
36
|
+
erb(get_template(:batch))
|
37
|
+
end
|
38
|
+
|
39
|
+
app.post "/batches/all" do
|
40
|
+
if params['delete']
|
41
|
+
drain_zset('batches') do |batches|
|
42
|
+
batches.each do |bid|
|
43
|
+
CanvasSync::JobBatches::Batch.cleanup_redis(bid)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
redirect "#{root_path}batches"
|
49
|
+
end
|
50
|
+
|
51
|
+
app.post "/batches/:bid" do
|
52
|
+
@bid = params[:bid]
|
53
|
+
@batch = CanvasSync::JobBatches::Batch.new(@bid)
|
54
|
+
|
55
|
+
if params['delete']
|
56
|
+
CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
|
57
|
+
end
|
58
|
+
|
59
|
+
redirect_with_query("#{root_path}batches")
|
60
|
+
end
|
61
|
+
|
62
|
+
# =============== POOLS =============== #
|
63
|
+
|
64
|
+
app.get "/pools" do
|
65
|
+
@count = (params['count'] || 25).to_i
|
66
|
+
@current_page, @total_size, @pools = page('pools', params['page'], @count)
|
67
|
+
@pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
|
68
|
+
|
69
|
+
erb(get_template(:pools))
|
70
|
+
end
|
71
|
+
|
72
|
+
app.get "/pools/:pid" do
|
73
|
+
@pid = params[:pid]
|
74
|
+
@pool = CanvasSync::JobBatches::Pool.new(@pid)
|
75
|
+
|
76
|
+
@count = (params['count'] || 25).to_i
|
77
|
+
@current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
|
78
|
+
@jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
|
79
|
+
|
80
|
+
erb(get_template(:pool))
|
81
|
+
end
|
82
|
+
|
83
|
+
app.post "/pools/all" do
|
84
|
+
if params['delete']
|
85
|
+
drain_zset('pools') do |pools|
|
86
|
+
pools.each do |pid|
|
87
|
+
CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
redirect "#{root_path}pools"
|
93
|
+
end
|
94
|
+
|
95
|
+
app.post "/pools/:pid" do
|
96
|
+
@pid = params[:pid]
|
97
|
+
@pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
|
98
|
+
|
99
|
+
if params['delete']
|
100
|
+
@pool.cleanup_redis
|
101
|
+
end
|
102
|
+
|
103
|
+
redirect_with_query("#{root_path}pools")
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
if defined?(::Sidekiq::Web)
|
110
|
+
::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
|
111
|
+
::Sidekiq::Web.tabs["Batches"] = "batches"
|
112
|
+
::Sidekiq::Web.tabs["Pools"] = "pools"
|
113
|
+
::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
|
114
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module CanvasSync::JobBatches::Sidekiq
|
4
|
+
module Web
|
5
|
+
module Helpers
|
6
|
+
VIEW_PATH = File.expand_path("../web/views", __dir__)
|
7
|
+
|
8
|
+
module_function
|
9
|
+
|
10
|
+
def get_template(name)
|
11
|
+
File.open(File.join(VIEW_PATH, "#{name}.erb")).read
|
12
|
+
end
|
13
|
+
|
14
|
+
def drain_zset(key)
|
15
|
+
items, _ = Sidekiq.redis do |r|
|
16
|
+
r.multi do
|
17
|
+
r.zrange(key, 0, -1)
|
18
|
+
r.zremrangebyrank(key, 0, -1)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
yield items
|
22
|
+
end
|
23
|
+
|
24
|
+
def safe_relative_time(time)
|
25
|
+
time = parse_time(time)
|
26
|
+
relative_time(time)
|
27
|
+
end
|
28
|
+
|
29
|
+
def parse_time(time)
|
30
|
+
case time
|
31
|
+
when Time
|
32
|
+
time
|
33
|
+
when Integer, Float
|
34
|
+
Time.at(time)
|
35
|
+
else
|
36
|
+
Time.parse(time.to_s)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|