canvas_sync 0.17.29 → 0.17.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/job_batches/batch.rb +9 -23
- data/lib/canvas_sync/job_batches/callback.rb +4 -4
- data/lib/canvas_sync/job_batches/context_hash.rb +1 -1
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +2 -2
- data/lib/canvas_sync/job_batches/pool.rb +6 -6
- data/lib/canvas_sync/job_batches/redis_model.rb +1 -1
- data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +1 -1
- data/lib/canvas_sync/jobs/term_batches_job.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/spec/dummy/log/development.log +2069 -0
- data/spec/dummy/log/test.log +101899 -0
- data/spec/job_batching/batch_spec.rb +0 -6
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +3 -0
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +4 -0
- metadata +11 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef8cd570185754998d8c14cd67f87c6d60e3507f7996925bdc4a425cd7d43f1d
|
4
|
+
data.tar.gz: 931416870261971aaeccfc59bae91972c8d7a7277aab9da4ac18610c3352a7fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7262c43e7395e4c9de96a9d5c4688ff85299fcab3f97abcea719b6f73c36fb519c23ccf7a224605a8552d0e547fe135ecc055ac7f664d240f0cc1dd8a932e2bf
|
7
|
+
data.tar.gz: 524b36471a7b135856c2e3bd935e8ab7e057616c1bd3ed4a6d5e7a6d66c98f3236cf87d041f946a5e4ac149bef9830100619ba9e61568971f0e17898ae2f44f3
|
@@ -242,7 +242,7 @@ module CanvasSync
|
|
242
242
|
_, pending, failed, children, complete, parent_bid = redis do |r|
|
243
243
|
return unless r.exists?("BID-#{bid}")
|
244
244
|
|
245
|
-
r.multi do
|
245
|
+
r.multi do |r|
|
246
246
|
r.sadd("BID-#{bid}-failed", jid)
|
247
247
|
|
248
248
|
r.hincrby("BID-#{bid}", "pending", 0)
|
@@ -260,29 +260,19 @@ module CanvasSync
|
|
260
260
|
end
|
261
261
|
end
|
262
262
|
|
263
|
+
# Dead jobs are a Sidekiq feature.
|
264
|
+
# If this is called for a job, process_failed_job was also called
|
263
265
|
def process_dead_job(bid, jid)
|
264
|
-
_,
|
266
|
+
_, dead_count = redis do |r|
|
265
267
|
return unless r.exists?("BID-#{bid}")
|
266
268
|
|
267
|
-
r.multi do
|
269
|
+
r.multi do |r|
|
268
270
|
r.sadd("BID-#{bid}-dead", jid)
|
269
|
-
|
270
271
|
r.scard("BID-#{bid}-dead")
|
271
|
-
r.hincrby("BID-#{bid}", "children", 0)
|
272
|
-
r.scard("BID-#{bid}-batches-complete")
|
273
|
-
r.hget("BID-#{bid}", "parent_bid")
|
274
|
-
|
275
272
|
r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
|
276
273
|
end
|
277
274
|
end
|
278
275
|
|
279
|
-
if parent_bid
|
280
|
-
redis.multi do
|
281
|
-
r.sadd("BID-#{parent_bid}-dead", jid)
|
282
|
-
r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
|
283
|
-
end
|
284
|
-
end
|
285
|
-
|
286
276
|
enqueue_callbacks(:death, bid)
|
287
277
|
end
|
288
278
|
|
@@ -290,7 +280,7 @@ module CanvasSync
|
|
290
280
|
_, failed, pending, children, complete, success, parent_bid, keep_open = redis do |r|
|
291
281
|
return unless r.exists?("BID-#{bid}")
|
292
282
|
|
293
|
-
r.multi do
|
283
|
+
r.multi do |r|
|
294
284
|
r.srem("BID-#{bid}-failed", jid)
|
295
285
|
|
296
286
|
r.scard("BID-#{bid}-failed")
|
@@ -323,7 +313,7 @@ module CanvasSync
|
|
323
313
|
return unless r.exists?(batch_key)
|
324
314
|
return if r.hget(batch_key, 'keep_open') == 'true'
|
325
315
|
|
326
|
-
r.multi do
|
316
|
+
r.multi do |r|
|
327
317
|
r.smembers(callback_key)
|
328
318
|
r.hget(batch_key, "callback_queue")
|
329
319
|
r.hget(batch_key, "parent_bid")
|
@@ -442,12 +432,8 @@ module CanvasSync
|
|
442
432
|
class RedisProxy
|
443
433
|
def multi(*args, &block)
|
444
434
|
Batch.redis do |r|
|
445
|
-
|
446
|
-
|
447
|
-
block.call(r)
|
448
|
-
end
|
449
|
-
else
|
450
|
-
r.multi(*args, &block)
|
435
|
+
r.multi(*args) do |r|
|
436
|
+
block.call(r)
|
451
437
|
end
|
452
438
|
end
|
453
439
|
end
|
@@ -69,7 +69,7 @@ module CanvasSync
|
|
69
69
|
# This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
|
70
70
|
origin_bid = opts['origin']['for_bid']
|
71
71
|
_, pending, success_ran = Batch.redis do |r|
|
72
|
-
r.multi do
|
72
|
+
r.multi do |r|
|
73
73
|
r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
|
74
74
|
r.scard("BID-#{origin_bid}-pending_callbacks")
|
75
75
|
r.hget("BID-#{origin_bid}", "success")
|
@@ -88,7 +88,7 @@ module CanvasSync
|
|
88
88
|
return unless parent_bid
|
89
89
|
|
90
90
|
_, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
|
91
|
-
r.multi do
|
91
|
+
r.multi do |r|
|
92
92
|
r.sadd("BID-#{parent_bid}-batches-success", bid)
|
93
93
|
r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
|
94
94
|
r.scard("BID-#{parent_bid}-batches-success")
|
@@ -116,7 +116,7 @@ module CanvasSync
|
|
116
116
|
|
117
117
|
def complete(bid, status, parent_bid)
|
118
118
|
pending, children, success = Batch.redis do |r|
|
119
|
-
r.multi do
|
119
|
+
r.multi do |r|
|
120
120
|
r.hincrby("BID-#{bid}", "pending", 0)
|
121
121
|
r.hincrby("BID-#{bid}", "children", 0)
|
122
122
|
r.scard("BID-#{bid}-batches-success")
|
@@ -131,7 +131,7 @@ module CanvasSync
|
|
131
131
|
|
132
132
|
Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
133
133
|
_, _, complete, pending, children, failure = Batch.redis do |r|
|
134
|
-
r.multi do
|
134
|
+
r.multi do |r|
|
135
135
|
r.sadd("BID-#{parent_bid}-batches-complete", bid)
|
136
136
|
r.sadd("BID-#{parent_bid}-batches-failed", bid)
|
137
137
|
r.scard("BID-#{parent_bid}-batches-complete")
|
@@ -18,7 +18,7 @@ module CanvasSync
|
|
18
18
|
man_batch_id = SecureRandom.urlsafe_base64(10)
|
19
19
|
|
20
20
|
Batch.redis do |r|
|
21
|
-
r.multi do
|
21
|
+
r.multi do |r|
|
22
22
|
r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
|
23
23
|
r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
|
24
24
|
r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
|
@@ -87,7 +87,7 @@ module CanvasSync
|
|
87
87
|
|
88
88
|
def self.perform_next_sequence_job(man_batch_id)
|
89
89
|
root_bid, ordered = Batch.redis do |r|
|
90
|
-
r.multi do
|
90
|
+
r.multi do |r|
|
91
91
|
r.hget("MNGBID-#{man_batch_id}", "root_bid")
|
92
92
|
r.hget("MNGBID-#{man_batch_id}", "ordered")
|
93
93
|
end
|
@@ -94,16 +94,16 @@ module CanvasSync
|
|
94
94
|
redis.hincrby(redis_key, "active_count", 0)
|
95
95
|
end
|
96
96
|
|
97
|
-
def pending_count
|
97
|
+
def pending_count(r = redis)
|
98
98
|
jobs_key = "#{redis_key}-jobs"
|
99
99
|
order = self.order || 'fifo'
|
100
100
|
case order.to_sym
|
101
101
|
when :fifo, :lifo
|
102
|
-
|
102
|
+
r.llen(jobs_key)
|
103
103
|
when :random
|
104
|
-
|
104
|
+
r.scard(jobs_key)
|
105
105
|
when :priority
|
106
|
-
|
106
|
+
r.zcard(jobs_key)
|
107
107
|
end
|
108
108
|
end
|
109
109
|
|
@@ -114,9 +114,9 @@ module CanvasSync
|
|
114
114
|
# Make sure this is loaded outside of the pipeline
|
115
115
|
self.order
|
116
116
|
|
117
|
-
redis.multi do
|
117
|
+
redis.multi do |r|
|
118
118
|
r.hincrby(redis_key, "active_count", -1)
|
119
|
-
self.pending_count
|
119
|
+
self.pending_count(r)
|
120
120
|
end
|
121
121
|
end
|
122
122
|
|
@@ -22,7 +22,7 @@ module CanvasSync
|
|
22
22
|
JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
|
23
23
|
b.description = "TermBatchJob(#{term_id}) Root"
|
24
24
|
b.context = local_context
|
25
|
-
b.on(:success, "#{self.class.to_s}.batch_finished")
|
25
|
+
b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
|
26
26
|
end
|
27
27
|
end
|
28
28
|
else
|
data/lib/canvas_sync/version.rb
CHANGED