canvas_sync 0.17.23 → 0.17.27
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/canvas_sync/importers/bulk_importer.rb +1 -1
- data/lib/canvas_sync/job_batches/batch.rb +17 -7
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +7 -7
- data/lib/canvas_sync/job_batches/chain_builder.rb +39 -6
- data/lib/canvas_sync/job_batches/context_hash.rb +4 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +12 -9
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +62 -25
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +9 -3
- data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/sidekiq.rb +7 -7
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +11 -1
- data/lib/canvas_sync/jobs/sync_terms_job.rb +1 -17
- data/lib/canvas_sync/jobs/term_batches_job.rb +50 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +31 -8
- data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb +13 -1
- data/spec/dummy/log/test.log +31707 -0
- data/spec/job_batching/batch_aware_job_spec.rb +3 -3
- data/spec/job_batching/batch_spec.rb +1 -1
- data/spec/job_batching/sidekiq_spec.rb +2 -2
- data/spec/support/fixtures/reports/users.csv +3 -2
- data/spec/support/fixtures/reports/xlist.csv +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b85b5374a289fbab25718bb03812c9623065dc5c8b4e35fff67c79d220d7fda9
|
4
|
+
data.tar.gz: 9a8d01f7a45bfe58339c8ba8a808a29d655711945ec7cf27a8e8fcd02205f343
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 73bf2bd012676ff339ccb3f2c7a2802e0752bc5623ccd301fc808d15ce139673be955ae4aee2b9ee842b18c2235cdbc843f197090c00b4caf6f57c6975d5c81a
|
7
|
+
data.tar.gz: eed6e24b8f30396a2f09751d2b786527a7b039c8b6fa6fbdcc9bdf695449f6e90ca76e138aec9fefb3a905abdbefd45ca092a1e1b85b0f706f6ca0fc35ff9fde
|
@@ -65,7 +65,7 @@ module CanvasSync
|
|
65
65
|
}
|
66
66
|
|
67
67
|
row_buffer = nil
|
68
|
-
if defined?(User) && klass == User && csv_column_names.include?(
|
68
|
+
if defined?(User) && klass == User && csv_column_names.include?('user_id')
|
69
69
|
row_buffer = UserRowBuffer.new(&row_buffer_out)
|
70
70
|
else
|
71
71
|
row_buffer = NullRowBuffer.new(&row_buffer_out)
|
@@ -19,6 +19,8 @@ require_relative "./chain_builder"
|
|
19
19
|
|
20
20
|
module CanvasSync
|
21
21
|
module JobBatches
|
22
|
+
CURRENT_BATCH_THREAD_KEY = :job_batches_batch
|
23
|
+
|
22
24
|
class Batch
|
23
25
|
include RedisModel
|
24
26
|
|
@@ -89,7 +91,7 @@ module CanvasSync
|
|
89
91
|
raise NoBlockGivenError unless block_given?
|
90
92
|
|
91
93
|
if !@existing && !@initialized
|
92
|
-
parent_bid = Thread.current[
|
94
|
+
parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
|
93
95
|
|
94
96
|
redis.multi do |r|
|
95
97
|
r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
|
@@ -111,11 +113,11 @@ module CanvasSync
|
|
111
113
|
end
|
112
114
|
|
113
115
|
begin
|
114
|
-
parent = Thread.current[
|
115
|
-
Thread.current[
|
116
|
+
parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
117
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = self
|
116
118
|
yield
|
117
119
|
ensure
|
118
|
-
Thread.current[
|
120
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
|
119
121
|
end
|
120
122
|
|
121
123
|
nil
|
@@ -179,11 +181,11 @@ module CanvasSync
|
|
179
181
|
|
180
182
|
def self.with_batch(batch)
|
181
183
|
batch = self.new(batch) if batch.is_a?(String)
|
182
|
-
parent = Thread.current[
|
183
|
-
Thread.current[
|
184
|
+
parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
185
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
|
184
186
|
yield
|
185
187
|
ensure
|
186
|
-
Thread.current[
|
188
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
|
187
189
|
end
|
188
190
|
|
189
191
|
# Any Batches or Jobs created in the given block won't be assocaiated to the current batch
|
@@ -228,6 +230,14 @@ module CanvasSync
|
|
228
230
|
end
|
229
231
|
|
230
232
|
class << self
|
233
|
+
def current
|
234
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
235
|
+
end
|
236
|
+
|
237
|
+
def current_context
|
238
|
+
current&.context || {}
|
239
|
+
end
|
240
|
+
|
231
241
|
def process_failed_job(bid, jid)
|
232
242
|
_, pending, failed, children, complete, parent_bid = redis do |r|
|
233
243
|
return unless r.exists?("BID-#{bid}")
|
@@ -6,17 +6,17 @@ module CanvasSync
|
|
6
6
|
included do
|
7
7
|
around_perform do |job, block|
|
8
8
|
if (@bid) # This _must_ be @bid - not just bid
|
9
|
-
prev_batch = Thread.current[
|
9
|
+
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
10
10
|
begin
|
11
|
-
Thread.current[
|
11
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
|
12
12
|
block.call
|
13
|
-
Thread.current[
|
13
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
|
14
14
|
Batch.process_successful_job(@bid, job_id)
|
15
15
|
rescue
|
16
16
|
Batch.process_failed_job(@bid, job_id)
|
17
17
|
raise
|
18
18
|
ensure
|
19
|
-
Thread.current[
|
19
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
|
20
20
|
end
|
21
21
|
else
|
22
22
|
block.call
|
@@ -24,7 +24,7 @@ module CanvasSync
|
|
24
24
|
end
|
25
25
|
|
26
26
|
around_enqueue do |job, block|
|
27
|
-
if (batch = Thread.current[
|
27
|
+
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
|
28
28
|
@bid = batch.bid
|
29
29
|
batch.increment_job_queue(job_id) if @bid
|
30
30
|
end
|
@@ -33,11 +33,11 @@ module CanvasSync
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def bid
|
36
|
-
@bid || Thread.current[
|
36
|
+
@bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
|
37
37
|
end
|
38
38
|
|
39
39
|
def batch
|
40
|
-
Thread.current[
|
40
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
41
41
|
end
|
42
42
|
|
43
43
|
def batch_context
|
@@ -37,16 +37,29 @@ module CanvasSync
|
|
37
37
|
insert_at(-1, new_job)
|
38
38
|
end
|
39
39
|
|
40
|
-
def insert_at(position, new_jobs)
|
40
|
+
def insert_at(position, new_jobs, *args, &blk)
|
41
41
|
chain = self.class.get_chain_parameter(base_job)
|
42
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
43
|
+
new_jobs = build_job_hash(new_jobs, *args, &blk)
|
44
|
+
elsif args.length > 0
|
45
|
+
raise "Unexpected number of arguments"
|
46
|
+
end
|
42
47
|
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
43
48
|
chain.insert(position, *new_jobs)
|
44
49
|
end
|
45
50
|
|
46
|
-
def insert(new_jobs, **kwargs)
|
47
|
-
|
48
|
-
|
49
|
-
|
51
|
+
def insert(new_jobs, *args, **kwargs, &blk)
|
52
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
53
|
+
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
54
|
+
args << job_kwargs if job_kwargs.present?
|
55
|
+
new_jobs = build_job_hash(new_jobs, *args, &blk)
|
56
|
+
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
57
|
+
else
|
58
|
+
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
59
|
+
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
60
|
+
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
61
|
+
raise "Unexpected number of arguments" if args.length > 0
|
62
|
+
end
|
50
63
|
|
51
64
|
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
52
65
|
|
@@ -90,7 +103,6 @@ module CanvasSync
|
|
90
103
|
raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
|
91
104
|
return nil if matching_jobs.count == 0
|
92
105
|
|
93
|
-
|
94
106
|
job = matching_jobs[0][0]
|
95
107
|
job = self.class.new(job) unless job.is_a?(ChainBuilder)
|
96
108
|
job
|
@@ -116,8 +128,22 @@ module CanvasSync
|
|
116
128
|
end
|
117
129
|
end
|
118
130
|
|
131
|
+
def apply_block(&blk)
|
132
|
+
return unless blk.present?
|
133
|
+
instance_exec(&blk)
|
134
|
+
end
|
135
|
+
|
119
136
|
private
|
120
137
|
|
138
|
+
def build_job_hash(job, *params, &blk)
|
139
|
+
hsh = {
|
140
|
+
job: job,
|
141
|
+
parameters: params,
|
142
|
+
}
|
143
|
+
self.class.new(hsh).apply_block(&blk) if blk.present?
|
144
|
+
hsh
|
145
|
+
end
|
146
|
+
|
121
147
|
def find_matching_jobs(search_job, parent_job = self.base_job)
|
122
148
|
return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
|
123
149
|
|
@@ -150,6 +176,13 @@ module CanvasSync
|
|
150
176
|
end
|
151
177
|
|
152
178
|
class << self
|
179
|
+
def build(job, *args, &blk)
|
180
|
+
new(job).tap do |ch|
|
181
|
+
ch[:parameters] = args
|
182
|
+
ch.apply_block(&blk)
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
153
186
|
def _job_type_definitions
|
154
187
|
@job_type_definitions ||= {}
|
155
188
|
end
|
@@ -3,16 +3,19 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class ConcurrentBatchJob < BaseJob
|
6
|
+
def self.make_batch(sub_jobs, context: nil, &blk)
|
7
|
+
ManagedBatchJob.make_batch(
|
8
|
+
sub_jobs,
|
9
|
+
ordered: false,
|
10
|
+
concurrency: true,
|
11
|
+
context: context,
|
12
|
+
desc_prefix: 'ConcurrentBatchJob',
|
13
|
+
&blk
|
14
|
+
)
|
15
|
+
end
|
16
|
+
|
6
17
|
def perform(sub_jobs, context: nil)
|
7
|
-
|
8
|
-
b.description = "Concurrent Batch Root"
|
9
|
-
b.context = context
|
10
|
-
b.jobs do
|
11
|
-
sub_jobs.each do |j|
|
12
|
-
ChainBuilder.enqueue_job(j)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
18
|
+
self.class.make_batch(sub_jobs, context: context)
|
16
19
|
end
|
17
20
|
end
|
18
21
|
end
|
@@ -3,8 +3,8 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class ManagedBatchJob < BaseJob
|
6
|
-
def
|
7
|
-
|
6
|
+
def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
|
7
|
+
desc_prefix ||= ''
|
8
8
|
|
9
9
|
if concurrency == 0 || concurrency == nil || concurrency == true
|
10
10
|
concurrency = sub_jobs.count
|
@@ -14,38 +14,60 @@ module CanvasSync
|
|
14
14
|
|
15
15
|
root_batch = Batch.new
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
r.
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
17
|
+
if concurrency < sub_jobs.count
|
18
|
+
man_batch_id = SecureRandom.urlsafe_base64(10)
|
19
|
+
|
20
|
+
Batch.redis do |r|
|
21
|
+
r.multi do
|
22
|
+
r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
|
23
|
+
r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
|
24
|
+
r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
|
25
|
+
r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
|
26
|
+
|
27
|
+
mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
|
28
|
+
j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
|
29
|
+
j = ActiveJob::Arguments.serialize([j])
|
30
|
+
JSON.unparse(j)
|
31
|
+
end
|
32
|
+
if ordered
|
33
|
+
r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
|
34
|
+
else
|
35
|
+
r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
|
36
|
+
end
|
37
|
+
r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
|
33
38
|
end
|
34
|
-
r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
|
35
39
|
end
|
40
|
+
|
41
|
+
root_batch.allow_context_changes = (concurrency == 1)
|
42
|
+
root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
|
43
|
+
|
44
|
+
desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
|
36
45
|
end
|
37
46
|
|
38
|
-
root_batch.description = "Managed Batch Root (#{man_batch_id})"
|
39
|
-
root_batch.allow_context_changes = (concurrency == 1)
|
40
47
|
root_batch.context = context
|
41
|
-
root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
|
42
|
-
root_batch.jobs {}
|
43
48
|
|
44
|
-
|
45
|
-
|
49
|
+
blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
|
50
|
+
|
51
|
+
root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
|
52
|
+
|
53
|
+
if concurrency < sub_jobs.count
|
54
|
+
root_batch.jobs {}
|
55
|
+
concurrency.times do
|
56
|
+
perform_next_sequence_job(man_batch_id)
|
57
|
+
end
|
58
|
+
else
|
59
|
+
root_batch.jobs do
|
60
|
+
sub_jobs.each do |j|
|
61
|
+
ChainBuilder.enqueue_job(j)
|
62
|
+
end
|
63
|
+
end
|
46
64
|
end
|
47
65
|
end
|
48
66
|
|
67
|
+
def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
|
68
|
+
self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
|
69
|
+
end
|
70
|
+
|
49
71
|
def self.cleanup_redis(status, options)
|
50
72
|
man_batch_id = options['managed_batch_id']
|
51
73
|
Batch.redis do |r|
|
@@ -94,6 +116,21 @@ module CanvasSync
|
|
94
116
|
end
|
95
117
|
end
|
96
118
|
end
|
119
|
+
|
120
|
+
class ManagedBatchProxy
|
121
|
+
def initialize(real_batch)
|
122
|
+
@real_batch = real_batch
|
123
|
+
end
|
124
|
+
|
125
|
+
delegate_missing_to :real_batch
|
126
|
+
|
127
|
+
def jobs
|
128
|
+
raise "Managed Batches do not support calling .jobs directly!"
|
129
|
+
end
|
130
|
+
|
131
|
+
private
|
132
|
+
attr_reader :real_batch
|
133
|
+
end
|
97
134
|
end
|
98
135
|
end
|
99
136
|
end
|
@@ -3,14 +3,20 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class SerialBatchJob < BaseJob
|
6
|
-
def
|
7
|
-
ManagedBatchJob.
|
6
|
+
def self.make_batch(sub_jobs, context: nil, &blk)
|
7
|
+
ManagedBatchJob.make_batch(
|
8
8
|
sub_jobs,
|
9
|
-
context: context,
|
10
9
|
ordered: true,
|
11
10
|
concurrency: false,
|
11
|
+
context: context,
|
12
|
+
desc_prefix: 'SerialBatchJob',
|
13
|
+
&blk
|
12
14
|
)
|
13
15
|
end
|
16
|
+
|
17
|
+
def perform(sub_jobs, context: nil)
|
18
|
+
self.class.make_batch(sub_jobs, context: context)
|
19
|
+
end
|
14
20
|
end
|
15
21
|
end
|
16
22
|
end
|
@@ -8,11 +8,11 @@ module CanvasSync
|
|
8
8
|
module Sidekiq
|
9
9
|
module WorkerExtension
|
10
10
|
def bid
|
11
|
-
Thread.current[
|
11
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].bid
|
12
12
|
end
|
13
13
|
|
14
14
|
def batch
|
15
|
-
Thread.current[
|
15
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
16
16
|
end
|
17
17
|
|
18
18
|
def batch_context
|
@@ -42,7 +42,7 @@ module CanvasSync
|
|
42
42
|
|
43
43
|
class ClientMiddleware
|
44
44
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
45
|
-
if (batch = Thread.current[
|
45
|
+
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
|
46
46
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
47
47
|
end
|
48
48
|
yield
|
@@ -57,17 +57,17 @@ module CanvasSync
|
|
57
57
|
class ServerMiddleware
|
58
58
|
def call(_worker, msg, _queue)
|
59
59
|
if (bid = msg['bid'])
|
60
|
-
prev_batch = Thread.current[
|
60
|
+
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
61
61
|
begin
|
62
|
-
Thread.current[
|
62
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
|
63
63
|
yield
|
64
|
-
Thread.current[
|
64
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
|
65
65
|
Batch.process_successful_job(bid, msg['jid'])
|
66
66
|
rescue
|
67
67
|
Batch.process_failed_job(bid, msg['jid'])
|
68
68
|
raise
|
69
69
|
ensure
|
70
|
-
Thread.current[
|
70
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
|
71
71
|
end
|
72
72
|
else
|
73
73
|
yield
|
@@ -14,6 +14,12 @@ module CanvasSync
|
|
14
14
|
globals[:updated_after] = last_batch&.started_at&.iso8601
|
15
15
|
end
|
16
16
|
|
17
|
+
# Refuse to run syncs of the same genre if there is a running full sync
|
18
|
+
if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
|
19
|
+
Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
17
23
|
if should_full_sync?(globals[:full_sync_every])
|
18
24
|
globals[:updated_after] = nil
|
19
25
|
end
|
@@ -25,6 +31,10 @@ module CanvasSync
|
|
25
31
|
status: 'processing',
|
26
32
|
)
|
27
33
|
|
34
|
+
globals[:batch_genre] = genre
|
35
|
+
globals[:batch_start_time] = sync_batch.started_at.iso8601
|
36
|
+
globals[:sync_batch_id] = sync_batch.id
|
37
|
+
|
28
38
|
JobBatches::Batch.new.tap do |b|
|
29
39
|
b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
|
30
40
|
b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
|
@@ -62,7 +72,7 @@ module CanvasSync
|
|
62
72
|
end
|
63
73
|
|
64
74
|
def last_full_sync_record
|
65
|
-
@last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
|
75
|
+
@last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
|
66
76
|
end
|
67
77
|
|
68
78
|
def last_full_sync
|
@@ -23,23 +23,7 @@ module CanvasSync
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
|
-
context = options[:context] || {}
|
28
|
-
if options[:term_scope]
|
29
|
-
Term.send(options[:term_scope]).find_each.map do |term|
|
30
|
-
local_context = context.merge(canvas_term_id: get_term_id(term))
|
31
|
-
JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
|
32
|
-
end
|
33
|
-
else
|
34
|
-
JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
protected
|
40
|
-
|
41
|
-
def get_term_id(term)
|
42
|
-
term.try(:canvas_id) || term.canvas_term_id
|
26
|
+
TermBatchesJob.perform_now(options)
|
43
27
|
end
|
44
28
|
end
|
45
29
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module CanvasSync
|
2
|
+
module Jobs
|
3
|
+
class TermBatchesJob < CanvasSync::Job
|
4
|
+
def perform(options)
|
5
|
+
if (jobs = options[:sub_jobs]).present?
|
6
|
+
context = options[:context] || {}
|
7
|
+
if options[:term_scope]
|
8
|
+
Term.send(options[:term_scope]).find_each.map do |term|
|
9
|
+
term_id = get_term_id(term)
|
10
|
+
local_context = context.merge(canvas_term_id: term_id)
|
11
|
+
|
12
|
+
# Override the delta-syncing date if:
|
13
|
+
# 1. the Term hasn't been synced before or
|
14
|
+
# 2. the Term underwent a period of not syncing
|
15
|
+
term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
|
16
|
+
if batch_context[:updated_after]
|
17
|
+
if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
|
18
|
+
local_context[:updated_after] = term_last_sync.presence
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
|
23
|
+
b.description = "TermBatchJob(#{term.canvas_id}) Root"
|
24
|
+
b.context = local_context
|
25
|
+
b.on(:success, "#{self.class.to_s}.batch_finished")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
else
|
29
|
+
JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.batch_finished(status, opts)
|
35
|
+
ctx = JobBatches::Batch.current_context
|
36
|
+
term_id = ctx[:canvas_term_id]
|
37
|
+
CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.last_sync_key(term_id)
|
41
|
+
ctx = JobBatches::Batch.current_context
|
42
|
+
"#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
|
43
|
+
end
|
44
|
+
|
45
|
+
def get_term_id(term)
|
46
|
+
term.try(:canvas_id) || term.canvas_term_id
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/canvas_sync/version.rb
CHANGED
data/lib/canvas_sync.rb
CHANGED
@@ -80,6 +80,7 @@ module CanvasSync
|
|
80
80
|
].freeze
|
81
81
|
|
82
82
|
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
|
83
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::TermBatchesJob, :sub_jobs)
|
83
84
|
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
|
84
85
|
|
85
86
|
class << self
|
@@ -130,12 +131,8 @@ module CanvasSync
|
|
130
131
|
models,
|
131
132
|
term_scope: nil,
|
132
133
|
term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
|
133
|
-
|
134
|
-
|
135
|
-
updated_after: nil,
|
136
|
-
full_sync_every: nil,
|
137
|
-
batch_genre: nil,
|
138
|
-
options: {}
|
134
|
+
options: {},
|
135
|
+
**kwargs
|
139
136
|
) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
|
140
137
|
return unless models.present?
|
141
138
|
models.map! &:to_s
|
@@ -155,7 +152,7 @@ module CanvasSync
|
|
155
152
|
context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
|
156
153
|
}.with_indifferent_access
|
157
154
|
|
158
|
-
root_chain =
|
155
|
+
root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
|
159
156
|
concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
|
160
157
|
root_chain << concurrent_root_chain
|
161
158
|
current_chain = concurrent_root_chain
|
@@ -211,6 +208,20 @@ module CanvasSync
|
|
211
208
|
# Wrap it all up
|
212
209
|
###############################
|
213
210
|
|
211
|
+
root_chain
|
212
|
+
end
|
213
|
+
|
214
|
+
def base_canvas_sync_chain(
|
215
|
+
legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
|
216
|
+
account_id: nil, # legacy/non PandaPal apps
|
217
|
+
updated_after: nil,
|
218
|
+
full_sync_every: nil,
|
219
|
+
batch_genre: nil,
|
220
|
+
globals: {},
|
221
|
+
&blk
|
222
|
+
)
|
223
|
+
root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
|
224
|
+
|
214
225
|
global_options = {
|
215
226
|
legacy_support: legacy_support,
|
216
227
|
updated_after: updated_after,
|
@@ -218,10 +229,12 @@ module CanvasSync
|
|
218
229
|
batch_genre: batch_genre,
|
219
230
|
}
|
220
231
|
global_options[:account_id] = account_id if account_id.present?
|
221
|
-
global_options.merge!(
|
232
|
+
global_options.merge!(globals) if globals
|
222
233
|
|
223
234
|
root_chain.params[1] = global_options
|
224
235
|
|
236
|
+
root_chain.apply_block(&blk)
|
237
|
+
|
225
238
|
root_chain
|
226
239
|
end
|
227
240
|
|
@@ -307,5 +320,15 @@ module CanvasSync
|
|
307
320
|
return if invalid.empty?
|
308
321
|
raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
|
309
322
|
end
|
323
|
+
|
324
|
+
def redis(*args, &blk)
|
325
|
+
JobBatches::Batch.redis(*args, &blk)
|
326
|
+
end
|
327
|
+
|
328
|
+
def redis_prefix
|
329
|
+
pfx = "cs"
|
330
|
+
pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
|
331
|
+
pfx
|
332
|
+
end
|
310
333
|
end
|
311
334
|
end
|