canvas_sync 0.17.23 → 0.17.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/importers/bulk_importer.rb +1 -1
- data/lib/canvas_sync/job_batches/batch.rb +17 -7
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +7 -7
- data/lib/canvas_sync/job_batches/chain_builder.rb +39 -6
- data/lib/canvas_sync/job_batches/context_hash.rb +4 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +12 -9
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +62 -25
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +9 -3
- data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/sidekiq.rb +7 -7
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +11 -1
- data/lib/canvas_sync/jobs/sync_terms_job.rb +1 -17
- data/lib/canvas_sync/jobs/term_batches_job.rb +50 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +31 -8
- data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb +13 -1
- data/spec/dummy/log/test.log +31707 -0
- data/spec/job_batching/batch_aware_job_spec.rb +3 -3
- data/spec/job_batching/batch_spec.rb +1 -1
- data/spec/job_batching/sidekiq_spec.rb +2 -2
- data/spec/support/fixtures/reports/users.csv +3 -2
- data/spec/support/fixtures/reports/xlist.csv +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b85b5374a289fbab25718bb03812c9623065dc5c8b4e35fff67c79d220d7fda9
|
4
|
+
data.tar.gz: 9a8d01f7a45bfe58339c8ba8a808a29d655711945ec7cf27a8e8fcd02205f343
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 73bf2bd012676ff339ccb3f2c7a2802e0752bc5623ccd301fc808d15ce139673be955ae4aee2b9ee842b18c2235cdbc843f197090c00b4caf6f57c6975d5c81a
|
7
|
+
data.tar.gz: eed6e24b8f30396a2f09751d2b786527a7b039c8b6fa6fbdcc9bdf695449f6e90ca76e138aec9fefb3a905abdbefd45ca092a1e1b85b0f706f6ca0fc35ff9fde
|
@@ -65,7 +65,7 @@ module CanvasSync
|
|
65
65
|
}
|
66
66
|
|
67
67
|
row_buffer = nil
|
68
|
-
if defined?(User) && klass == User && csv_column_names.include?(
|
68
|
+
if defined?(User) && klass == User && csv_column_names.include?('user_id')
|
69
69
|
row_buffer = UserRowBuffer.new(&row_buffer_out)
|
70
70
|
else
|
71
71
|
row_buffer = NullRowBuffer.new(&row_buffer_out)
|
@@ -19,6 +19,8 @@ require_relative "./chain_builder"
|
|
19
19
|
|
20
20
|
module CanvasSync
|
21
21
|
module JobBatches
|
22
|
+
CURRENT_BATCH_THREAD_KEY = :job_batches_batch
|
23
|
+
|
22
24
|
class Batch
|
23
25
|
include RedisModel
|
24
26
|
|
@@ -89,7 +91,7 @@ module CanvasSync
|
|
89
91
|
raise NoBlockGivenError unless block_given?
|
90
92
|
|
91
93
|
if !@existing && !@initialized
|
92
|
-
parent_bid = Thread.current[
|
94
|
+
parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
|
93
95
|
|
94
96
|
redis.multi do |r|
|
95
97
|
r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
|
@@ -111,11 +113,11 @@ module CanvasSync
|
|
111
113
|
end
|
112
114
|
|
113
115
|
begin
|
114
|
-
parent = Thread.current[
|
115
|
-
Thread.current[
|
116
|
+
parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
117
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = self
|
116
118
|
yield
|
117
119
|
ensure
|
118
|
-
Thread.current[
|
120
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
|
119
121
|
end
|
120
122
|
|
121
123
|
nil
|
@@ -179,11 +181,11 @@ module CanvasSync
|
|
179
181
|
|
180
182
|
def self.with_batch(batch)
|
181
183
|
batch = self.new(batch) if batch.is_a?(String)
|
182
|
-
parent = Thread.current[
|
183
|
-
Thread.current[
|
184
|
+
parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
185
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
|
184
186
|
yield
|
185
187
|
ensure
|
186
|
-
Thread.current[
|
188
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
|
187
189
|
end
|
188
190
|
|
189
191
|
# Any Batches or Jobs created in the given block won't be assocaiated to the current batch
|
@@ -228,6 +230,14 @@ module CanvasSync
|
|
228
230
|
end
|
229
231
|
|
230
232
|
class << self
|
233
|
+
def current
|
234
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
235
|
+
end
|
236
|
+
|
237
|
+
def current_context
|
238
|
+
current&.context || {}
|
239
|
+
end
|
240
|
+
|
231
241
|
def process_failed_job(bid, jid)
|
232
242
|
_, pending, failed, children, complete, parent_bid = redis do |r|
|
233
243
|
return unless r.exists?("BID-#{bid}")
|
@@ -6,17 +6,17 @@ module CanvasSync
|
|
6
6
|
included do
|
7
7
|
around_perform do |job, block|
|
8
8
|
if (@bid) # This _must_ be @bid - not just bid
|
9
|
-
prev_batch = Thread.current[
|
9
|
+
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
10
10
|
begin
|
11
|
-
Thread.current[
|
11
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
|
12
12
|
block.call
|
13
|
-
Thread.current[
|
13
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
|
14
14
|
Batch.process_successful_job(@bid, job_id)
|
15
15
|
rescue
|
16
16
|
Batch.process_failed_job(@bid, job_id)
|
17
17
|
raise
|
18
18
|
ensure
|
19
|
-
Thread.current[
|
19
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
|
20
20
|
end
|
21
21
|
else
|
22
22
|
block.call
|
@@ -24,7 +24,7 @@ module CanvasSync
|
|
24
24
|
end
|
25
25
|
|
26
26
|
around_enqueue do |job, block|
|
27
|
-
if (batch = Thread.current[
|
27
|
+
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
|
28
28
|
@bid = batch.bid
|
29
29
|
batch.increment_job_queue(job_id) if @bid
|
30
30
|
end
|
@@ -33,11 +33,11 @@ module CanvasSync
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def bid
|
36
|
-
@bid || Thread.current[
|
36
|
+
@bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
|
37
37
|
end
|
38
38
|
|
39
39
|
def batch
|
40
|
-
Thread.current[
|
40
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
41
41
|
end
|
42
42
|
|
43
43
|
def batch_context
|
@@ -37,16 +37,29 @@ module CanvasSync
|
|
37
37
|
insert_at(-1, new_job)
|
38
38
|
end
|
39
39
|
|
40
|
-
def insert_at(position, new_jobs)
|
40
|
+
def insert_at(position, new_jobs, *args, &blk)
|
41
41
|
chain = self.class.get_chain_parameter(base_job)
|
42
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
43
|
+
new_jobs = build_job_hash(new_jobs, *args, &blk)
|
44
|
+
elsif args.length > 0
|
45
|
+
raise "Unexpected number of arguments"
|
46
|
+
end
|
42
47
|
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
43
48
|
chain.insert(position, *new_jobs)
|
44
49
|
end
|
45
50
|
|
46
|
-
def insert(new_jobs, **kwargs)
|
47
|
-
|
48
|
-
|
49
|
-
|
51
|
+
def insert(new_jobs, *args, **kwargs, &blk)
|
52
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
53
|
+
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
54
|
+
args << job_kwargs if job_kwargs.present?
|
55
|
+
new_jobs = build_job_hash(new_jobs, *args, &blk)
|
56
|
+
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
57
|
+
else
|
58
|
+
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
59
|
+
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
60
|
+
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
61
|
+
raise "Unexpected number of arguments" if args.length > 0
|
62
|
+
end
|
50
63
|
|
51
64
|
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
52
65
|
|
@@ -90,7 +103,6 @@ module CanvasSync
|
|
90
103
|
raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
|
91
104
|
return nil if matching_jobs.count == 0
|
92
105
|
|
93
|
-
|
94
106
|
job = matching_jobs[0][0]
|
95
107
|
job = self.class.new(job) unless job.is_a?(ChainBuilder)
|
96
108
|
job
|
@@ -116,8 +128,22 @@ module CanvasSync
|
|
116
128
|
end
|
117
129
|
end
|
118
130
|
|
131
|
+
def apply_block(&blk)
|
132
|
+
return unless blk.present?
|
133
|
+
instance_exec(&blk)
|
134
|
+
end
|
135
|
+
|
119
136
|
private
|
120
137
|
|
138
|
+
def build_job_hash(job, *params, &blk)
|
139
|
+
hsh = {
|
140
|
+
job: job,
|
141
|
+
parameters: params,
|
142
|
+
}
|
143
|
+
self.class.new(hsh).apply_block(&blk) if blk.present?
|
144
|
+
hsh
|
145
|
+
end
|
146
|
+
|
121
147
|
def find_matching_jobs(search_job, parent_job = self.base_job)
|
122
148
|
return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
|
123
149
|
|
@@ -150,6 +176,13 @@ module CanvasSync
|
|
150
176
|
end
|
151
177
|
|
152
178
|
class << self
|
179
|
+
def build(job, *args, &blk)
|
180
|
+
new(job).tap do |ch|
|
181
|
+
ch[:parameters] = args
|
182
|
+
ch.apply_block(&blk)
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
153
186
|
def _job_type_definitions
|
154
187
|
@job_type_definitions ||= {}
|
155
188
|
end
|
@@ -3,16 +3,19 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class ConcurrentBatchJob < BaseJob
|
6
|
+
def self.make_batch(sub_jobs, context: nil, &blk)
|
7
|
+
ManagedBatchJob.make_batch(
|
8
|
+
sub_jobs,
|
9
|
+
ordered: false,
|
10
|
+
concurrency: true,
|
11
|
+
context: context,
|
12
|
+
desc_prefix: 'ConcurrentBatchJob',
|
13
|
+
&blk
|
14
|
+
)
|
15
|
+
end
|
16
|
+
|
6
17
|
def perform(sub_jobs, context: nil)
|
7
|
-
|
8
|
-
b.description = "Concurrent Batch Root"
|
9
|
-
b.context = context
|
10
|
-
b.jobs do
|
11
|
-
sub_jobs.each do |j|
|
12
|
-
ChainBuilder.enqueue_job(j)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
18
|
+
self.class.make_batch(sub_jobs, context: context)
|
16
19
|
end
|
17
20
|
end
|
18
21
|
end
|
@@ -3,8 +3,8 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class ManagedBatchJob < BaseJob
|
6
|
-
def
|
7
|
-
|
6
|
+
def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
|
7
|
+
desc_prefix ||= ''
|
8
8
|
|
9
9
|
if concurrency == 0 || concurrency == nil || concurrency == true
|
10
10
|
concurrency = sub_jobs.count
|
@@ -14,38 +14,60 @@ module CanvasSync
|
|
14
14
|
|
15
15
|
root_batch = Batch.new
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
r.
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
17
|
+
if concurrency < sub_jobs.count
|
18
|
+
man_batch_id = SecureRandom.urlsafe_base64(10)
|
19
|
+
|
20
|
+
Batch.redis do |r|
|
21
|
+
r.multi do
|
22
|
+
r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
|
23
|
+
r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
|
24
|
+
r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
|
25
|
+
r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
|
26
|
+
|
27
|
+
mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
|
28
|
+
j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
|
29
|
+
j = ActiveJob::Arguments.serialize([j])
|
30
|
+
JSON.unparse(j)
|
31
|
+
end
|
32
|
+
if ordered
|
33
|
+
r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
|
34
|
+
else
|
35
|
+
r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
|
36
|
+
end
|
37
|
+
r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
|
33
38
|
end
|
34
|
-
r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
|
35
39
|
end
|
40
|
+
|
41
|
+
root_batch.allow_context_changes = (concurrency == 1)
|
42
|
+
root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
|
43
|
+
|
44
|
+
desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
|
36
45
|
end
|
37
46
|
|
38
|
-
root_batch.description = "Managed Batch Root (#{man_batch_id})"
|
39
|
-
root_batch.allow_context_changes = (concurrency == 1)
|
40
47
|
root_batch.context = context
|
41
|
-
root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
|
42
|
-
root_batch.jobs {}
|
43
48
|
|
44
|
-
|
45
|
-
|
49
|
+
blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
|
50
|
+
|
51
|
+
root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
|
52
|
+
|
53
|
+
if concurrency < sub_jobs.count
|
54
|
+
root_batch.jobs {}
|
55
|
+
concurrency.times do
|
56
|
+
perform_next_sequence_job(man_batch_id)
|
57
|
+
end
|
58
|
+
else
|
59
|
+
root_batch.jobs do
|
60
|
+
sub_jobs.each do |j|
|
61
|
+
ChainBuilder.enqueue_job(j)
|
62
|
+
end
|
63
|
+
end
|
46
64
|
end
|
47
65
|
end
|
48
66
|
|
67
|
+
def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
|
68
|
+
self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
|
69
|
+
end
|
70
|
+
|
49
71
|
def self.cleanup_redis(status, options)
|
50
72
|
man_batch_id = options['managed_batch_id']
|
51
73
|
Batch.redis do |r|
|
@@ -94,6 +116,21 @@ module CanvasSync
|
|
94
116
|
end
|
95
117
|
end
|
96
118
|
end
|
119
|
+
|
120
|
+
class ManagedBatchProxy
|
121
|
+
def initialize(real_batch)
|
122
|
+
@real_batch = real_batch
|
123
|
+
end
|
124
|
+
|
125
|
+
delegate_missing_to :real_batch
|
126
|
+
|
127
|
+
def jobs
|
128
|
+
raise "Managed Batches do not support calling .jobs directly!"
|
129
|
+
end
|
130
|
+
|
131
|
+
private
|
132
|
+
attr_reader :real_batch
|
133
|
+
end
|
97
134
|
end
|
98
135
|
end
|
99
136
|
end
|
@@ -3,14 +3,20 @@ require_relative './base_job'
|
|
3
3
|
module CanvasSync
|
4
4
|
module JobBatches
|
5
5
|
class SerialBatchJob < BaseJob
|
6
|
-
def
|
7
|
-
ManagedBatchJob.
|
6
|
+
def self.make_batch(sub_jobs, context: nil, &blk)
|
7
|
+
ManagedBatchJob.make_batch(
|
8
8
|
sub_jobs,
|
9
|
-
context: context,
|
10
9
|
ordered: true,
|
11
10
|
concurrency: false,
|
11
|
+
context: context,
|
12
|
+
desc_prefix: 'SerialBatchJob',
|
13
|
+
&blk
|
12
14
|
)
|
13
15
|
end
|
16
|
+
|
17
|
+
def perform(sub_jobs, context: nil)
|
18
|
+
self.class.make_batch(sub_jobs, context: context)
|
19
|
+
end
|
14
20
|
end
|
15
21
|
end
|
16
22
|
end
|
@@ -8,11 +8,11 @@ module CanvasSync
|
|
8
8
|
module Sidekiq
|
9
9
|
module WorkerExtension
|
10
10
|
def bid
|
11
|
-
Thread.current[
|
11
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].bid
|
12
12
|
end
|
13
13
|
|
14
14
|
def batch
|
15
|
-
Thread.current[
|
15
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY]
|
16
16
|
end
|
17
17
|
|
18
18
|
def batch_context
|
@@ -42,7 +42,7 @@ module CanvasSync
|
|
42
42
|
|
43
43
|
class ClientMiddleware
|
44
44
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
45
|
-
if (batch = Thread.current[
|
45
|
+
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
|
46
46
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
47
47
|
end
|
48
48
|
yield
|
@@ -57,17 +57,17 @@ module CanvasSync
|
|
57
57
|
class ServerMiddleware
|
58
58
|
def call(_worker, msg, _queue)
|
59
59
|
if (bid = msg['bid'])
|
60
|
-
prev_batch = Thread.current[
|
60
|
+
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
61
61
|
begin
|
62
|
-
Thread.current[
|
62
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
|
63
63
|
yield
|
64
|
-
Thread.current[
|
64
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
|
65
65
|
Batch.process_successful_job(bid, msg['jid'])
|
66
66
|
rescue
|
67
67
|
Batch.process_failed_job(bid, msg['jid'])
|
68
68
|
raise
|
69
69
|
ensure
|
70
|
-
Thread.current[
|
70
|
+
Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
|
71
71
|
end
|
72
72
|
else
|
73
73
|
yield
|
@@ -14,6 +14,12 @@ module CanvasSync
|
|
14
14
|
globals[:updated_after] = last_batch&.started_at&.iso8601
|
15
15
|
end
|
16
16
|
|
17
|
+
# Refuse to run syncs of the same genre if there is a running full sync
|
18
|
+
if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
|
19
|
+
Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
|
20
|
+
return
|
21
|
+
end
|
22
|
+
|
17
23
|
if should_full_sync?(globals[:full_sync_every])
|
18
24
|
globals[:updated_after] = nil
|
19
25
|
end
|
@@ -25,6 +31,10 @@ module CanvasSync
|
|
25
31
|
status: 'processing',
|
26
32
|
)
|
27
33
|
|
34
|
+
globals[:batch_genre] = genre
|
35
|
+
globals[:batch_start_time] = sync_batch.started_at.iso8601
|
36
|
+
globals[:sync_batch_id] = sync_batch.id
|
37
|
+
|
28
38
|
JobBatches::Batch.new.tap do |b|
|
29
39
|
b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
|
30
40
|
b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
|
@@ -62,7 +72,7 @@ module CanvasSync
|
|
62
72
|
end
|
63
73
|
|
64
74
|
def last_full_sync_record
|
65
|
-
@last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
|
75
|
+
@last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
|
66
76
|
end
|
67
77
|
|
68
78
|
def last_full_sync
|
@@ -23,23 +23,7 @@ module CanvasSync
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
-
|
27
|
-
context = options[:context] || {}
|
28
|
-
if options[:term_scope]
|
29
|
-
Term.send(options[:term_scope]).find_each.map do |term|
|
30
|
-
local_context = context.merge(canvas_term_id: get_term_id(term))
|
31
|
-
JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
|
32
|
-
end
|
33
|
-
else
|
34
|
-
JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
protected
|
40
|
-
|
41
|
-
def get_term_id(term)
|
42
|
-
term.try(:canvas_id) || term.canvas_term_id
|
26
|
+
TermBatchesJob.perform_now(options)
|
43
27
|
end
|
44
28
|
end
|
45
29
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module CanvasSync
|
2
|
+
module Jobs
|
3
|
+
class TermBatchesJob < CanvasSync::Job
|
4
|
+
def perform(options)
|
5
|
+
if (jobs = options[:sub_jobs]).present?
|
6
|
+
context = options[:context] || {}
|
7
|
+
if options[:term_scope]
|
8
|
+
Term.send(options[:term_scope]).find_each.map do |term|
|
9
|
+
term_id = get_term_id(term)
|
10
|
+
local_context = context.merge(canvas_term_id: term_id)
|
11
|
+
|
12
|
+
# Override the delta-syncing date if:
|
13
|
+
# 1. the Term hasn't been synced before or
|
14
|
+
# 2. the Term underwent a period of not syncing
|
15
|
+
term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
|
16
|
+
if batch_context[:updated_after]
|
17
|
+
if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
|
18
|
+
local_context[:updated_after] = term_last_sync.presence
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
|
23
|
+
b.description = "TermBatchJob(#{term.canvas_id}) Root"
|
24
|
+
b.context = local_context
|
25
|
+
b.on(:success, "#{self.class.to_s}.batch_finished")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
else
|
29
|
+
JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.batch_finished(status, opts)
|
35
|
+
ctx = JobBatches::Batch.current_context
|
36
|
+
term_id = ctx[:canvas_term_id]
|
37
|
+
CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.last_sync_key(term_id)
|
41
|
+
ctx = JobBatches::Batch.current_context
|
42
|
+
"#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
|
43
|
+
end
|
44
|
+
|
45
|
+
def get_term_id(term)
|
46
|
+
term.try(:canvas_id) || term.canvas_term_id
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/canvas_sync/version.rb
CHANGED
data/lib/canvas_sync.rb
CHANGED
@@ -80,6 +80,7 @@ module CanvasSync
|
|
80
80
|
].freeze
|
81
81
|
|
82
82
|
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
|
83
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::TermBatchesJob, :sub_jobs)
|
83
84
|
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
|
84
85
|
|
85
86
|
class << self
|
@@ -130,12 +131,8 @@ module CanvasSync
|
|
130
131
|
models,
|
131
132
|
term_scope: nil,
|
132
133
|
term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
|
133
|
-
|
134
|
-
|
135
|
-
updated_after: nil,
|
136
|
-
full_sync_every: nil,
|
137
|
-
batch_genre: nil,
|
138
|
-
options: {}
|
134
|
+
options: {},
|
135
|
+
**kwargs
|
139
136
|
) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
|
140
137
|
return unless models.present?
|
141
138
|
models.map! &:to_s
|
@@ -155,7 +152,7 @@ module CanvasSync
|
|
155
152
|
context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
|
156
153
|
}.with_indifferent_access
|
157
154
|
|
158
|
-
root_chain =
|
155
|
+
root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
|
159
156
|
concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
|
160
157
|
root_chain << concurrent_root_chain
|
161
158
|
current_chain = concurrent_root_chain
|
@@ -211,6 +208,20 @@ module CanvasSync
|
|
211
208
|
# Wrap it all up
|
212
209
|
###############################
|
213
210
|
|
211
|
+
root_chain
|
212
|
+
end
|
213
|
+
|
214
|
+
def base_canvas_sync_chain(
|
215
|
+
legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
|
216
|
+
account_id: nil, # legacy/non PandaPal apps
|
217
|
+
updated_after: nil,
|
218
|
+
full_sync_every: nil,
|
219
|
+
batch_genre: nil,
|
220
|
+
globals: {},
|
221
|
+
&blk
|
222
|
+
)
|
223
|
+
root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
|
224
|
+
|
214
225
|
global_options = {
|
215
226
|
legacy_support: legacy_support,
|
216
227
|
updated_after: updated_after,
|
@@ -218,10 +229,12 @@ module CanvasSync
|
|
218
229
|
batch_genre: batch_genre,
|
219
230
|
}
|
220
231
|
global_options[:account_id] = account_id if account_id.present?
|
221
|
-
global_options.merge!(
|
232
|
+
global_options.merge!(globals) if globals
|
222
233
|
|
223
234
|
root_chain.params[1] = global_options
|
224
235
|
|
236
|
+
root_chain.apply_block(&blk)
|
237
|
+
|
225
238
|
root_chain
|
226
239
|
end
|
227
240
|
|
@@ -307,5 +320,15 @@ module CanvasSync
|
|
307
320
|
return if invalid.empty?
|
308
321
|
raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
|
309
322
|
end
|
323
|
+
|
324
|
+
def redis(*args, &blk)
|
325
|
+
JobBatches::Batch.redis(*args, &blk)
|
326
|
+
end
|
327
|
+
|
328
|
+
def redis_prefix
|
329
|
+
pfx = "cs"
|
330
|
+
pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
|
331
|
+
pfx
|
332
|
+
end
|
310
333
|
end
|
311
334
|
end
|