canvas_sync 0.17.23.beta8 → 0.17.27.beta1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9bed2b1c5be095c602aca19968b0155ff48026a1c5136cb65b5f31e5decbcb40
4
- data.tar.gz: ae6112e2694f0118a43619cf3de053d7eee7ff1873e0dface27b0b3558727132
3
+ metadata.gz: 413b8669b85123f239a5917982347c9cb90bd56229c42e11a92ce8bc619171f5
4
+ data.tar.gz: b99de51cad35d7c894ed54ab8910cbe6d0b594cadab814683396350dbbc4b95a
5
5
  SHA512:
6
- metadata.gz: 6261b328341a102a9ea6c5f91315444604a3556ccbc040fb38122d18db557bd2c5d2cbc7e5b3c74a52d0d48eb0315793232a3cdff0be21964af1c2617d4ae22d
7
- data.tar.gz: 458a2c7b422c2a769ec961bd75bd8f43e5dde63c9231bbd4aab6e2a400ec79ee74d9dc12d93cdbd2cc26ef69bb3611972b84956f55cfc2e2a89a4fc22ec2eafb
6
+ metadata.gz: e78f116e14b49c88fb04d95b33623c99a31c6a09735491361b0c287a2d48fff2bf061710f5715854b8601b7eb0c1aa3de9bfd93d87d08f71c260eac830a08fce
7
+ data.tar.gz: 361de97175217d570b0e60f4b3737a38d685ea4404d568ad35df281f495f08288284d734a02e2f8c5809c2a81e3863bb3030a217ce5d5bab71ca60d889eff31c
@@ -65,7 +65,7 @@ module CanvasSync
65
65
  }
66
66
 
67
67
  row_buffer = nil
68
- if defined?(User) && klass == User && csv_column_names.include?(:user_id)
68
+ if defined?(User) && klass == User && csv_column_names.include?('user_id')
69
69
  row_buffer = UserRowBuffer.new(&row_buffer_out)
70
70
  else
71
71
  row_buffer = NullRowBuffer.new(&row_buffer_out)
@@ -19,6 +19,8 @@ require_relative "./chain_builder"
19
19
 
20
20
  module CanvasSync
21
21
  module JobBatches
22
+ CURRENT_BATCH_THREAD_KEY = :job_batches_batch
23
+
22
24
  class Batch
23
25
  include RedisModel
24
26
 
@@ -89,7 +91,7 @@ module CanvasSync
89
91
  raise NoBlockGivenError unless block_given?
90
92
 
91
93
  if !@existing && !@initialized
92
- parent_bid = Thread.current[:batch]&.bid
94
+ parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
93
95
 
94
96
  redis.multi do |r|
95
97
  r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
@@ -111,11 +113,11 @@ module CanvasSync
111
113
  end
112
114
 
113
115
  begin
114
- parent = Thread.current[:batch]
115
- Thread.current[:batch] = self
116
+ parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
117
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = self
116
118
  yield
117
119
  ensure
118
- Thread.current[:batch] = parent
120
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
119
121
  end
120
122
 
121
123
  nil
@@ -179,11 +181,11 @@ module CanvasSync
179
181
 
180
182
  def self.with_batch(batch)
181
183
  batch = self.new(batch) if batch.is_a?(String)
182
- parent = Thread.current[:batch]
183
- Thread.current[:batch] = batch
184
+ parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
185
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
184
186
  yield
185
187
  ensure
186
- Thread.current[:batch] = parent
188
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
187
189
  end
188
190
 
189
191
  # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
@@ -228,6 +230,14 @@ module CanvasSync
228
230
  end
229
231
 
230
232
  class << self
233
+ def current
234
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
235
+ end
236
+
237
+ def current_context
238
+ current&.context || {}
239
+ end
240
+
231
241
  def process_failed_job(bid, jid)
232
242
  _, pending, failed, children, complete, parent_bid = redis do |r|
233
243
  return unless r.exists?("BID-#{bid}")
@@ -6,17 +6,17 @@ module CanvasSync
6
6
  included do
7
7
  around_perform do |job, block|
8
8
  if (@bid) # This _must_ be @bid - not just bid
9
- prev_batch = Thread.current[:batch]
9
+ prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
10
10
  begin
11
- Thread.current[:batch] = Batch.new(@bid)
11
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
12
12
  block.call
13
- Thread.current[:batch].save_context_changes
13
+ Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
14
14
  Batch.process_successful_job(@bid, job_id)
15
15
  rescue
16
16
  Batch.process_failed_job(@bid, job_id)
17
17
  raise
18
18
  ensure
19
- Thread.current[:batch] = prev_batch
19
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
20
20
  end
21
21
  else
22
22
  block.call
@@ -24,7 +24,7 @@ module CanvasSync
24
24
  end
25
25
 
26
26
  around_enqueue do |job, block|
27
- if (batch = Thread.current[:batch])
27
+ if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
28
28
  @bid = batch.bid
29
29
  batch.increment_job_queue(job_id) if @bid
30
30
  end
@@ -33,11 +33,11 @@ module CanvasSync
33
33
  end
34
34
 
35
35
  def bid
36
- @bid || Thread.current[:batch]&.bid
36
+ @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
37
37
  end
38
38
 
39
39
  def batch
40
- Thread.current[:batch]
40
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
41
41
  end
42
42
 
43
43
  def batch_context
@@ -37,16 +37,29 @@ module CanvasSync
37
37
  insert_at(-1, new_job)
38
38
  end
39
39
 
40
- def insert_at(position, new_jobs)
40
+ def insert_at(position, new_jobs, *args, &blk)
41
41
  chain = self.class.get_chain_parameter(base_job)
42
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
43
+ new_jobs = build_job_hash(new_jobs, *args, &blk)
44
+ elsif args.length > 0
45
+ raise "Unexpected number of arguments"
46
+ end
42
47
  new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
43
48
  chain.insert(position, *new_jobs)
44
49
  end
45
50
 
46
- def insert(new_jobs, **kwargs)
47
- invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
48
- raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
49
- raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
51
+ def insert(new_jobs, *args, **kwargs, &blk)
52
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
53
+ job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
54
+ args << job_kwargs if job_kwargs.present?
55
+ new_jobs = build_job_hash(new_jobs, *args, &blk)
56
+ kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
57
+ else
58
+ invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
59
+ raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
60
+ raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
61
+ raise "Unexpected number of arguments" if args.length > 0
62
+ end
50
63
 
51
64
  new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
52
65
 
@@ -90,7 +103,6 @@ module CanvasSync
90
103
  raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
91
104
  return nil if matching_jobs.count == 0
92
105
 
93
-
94
106
  job = matching_jobs[0][0]
95
107
  job = self.class.new(job) unless job.is_a?(ChainBuilder)
96
108
  job
@@ -116,8 +128,22 @@ module CanvasSync
116
128
  end
117
129
  end
118
130
 
131
+ def apply_block(&blk)
132
+ return unless blk.present?
133
+ instance_exec(&blk)
134
+ end
135
+
119
136
  private
120
137
 
138
+ def build_job_hash(job, *params, &blk)
139
+ hsh = {
140
+ job: job,
141
+ parameters: params,
142
+ }
143
+ self.class.new(hsh).apply_block(&blk) if blk.present?
144
+ hsh
145
+ end
146
+
121
147
  def find_matching_jobs(search_job, parent_job = self.base_job)
122
148
  return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
123
149
 
@@ -150,6 +176,13 @@ module CanvasSync
150
176
  end
151
177
 
152
178
  class << self
179
+ def build(job, *args, &blk)
180
+ new(job).tap do |ch|
181
+ ch[:parameters] = args
182
+ ch.apply_block(&blk)
183
+ end
184
+ end
185
+
153
186
  def _job_type_definitions
154
187
  @job_type_definitions ||= {}
155
188
  end
@@ -95,6 +95,10 @@ module CanvasSync
95
95
  @flattened = flattened.with_indifferent_access
96
96
  end
97
97
 
98
+ def to_h
99
+ flatten
100
+ end
101
+
98
102
  private
99
103
 
100
104
  def get_parent_hash(bid)
@@ -3,16 +3,19 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class ConcurrentBatchJob < BaseJob
6
+ def self.make_batch(sub_jobs, context: nil, &blk)
7
+ ManagedBatchJob.make_batch(
8
+ sub_jobs,
9
+ ordered: false,
10
+ concurrency: true,
11
+ context: context,
12
+ desc_prefix: 'ConcurrentBatchJob',
13
+ &blk
14
+ )
15
+ end
16
+
6
17
  def perform(sub_jobs, context: nil)
7
- Batch.new.tap do |b|
8
- b.description = "Concurrent Batch Root"
9
- b.context = context
10
- b.jobs do
11
- sub_jobs.each do |j|
12
- ChainBuilder.enqueue_job(j)
13
- end
14
- end
15
- end
18
+ self.class.make_batch(sub_jobs, context: context)
16
19
  end
17
20
  end
18
21
  end
@@ -3,8 +3,8 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class ManagedBatchJob < BaseJob
6
- def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
7
- man_batch_id = SecureRandom.urlsafe_base64(10)
6
+ def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
7
+ desc_prefix ||= ''
8
8
 
9
9
  if concurrency == 0 || concurrency == nil || concurrency == true
10
10
  concurrency = sub_jobs.count
@@ -14,38 +14,60 @@ module CanvasSync
14
14
 
15
15
  root_batch = Batch.new
16
16
 
17
- Batch.redis do |r|
18
- r.multi do
19
- r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
20
- r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
21
- r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
22
- r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
23
-
24
- mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
25
- j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
26
- j = ActiveJob::Arguments.serialize([j])
27
- JSON.unparse(j)
28
- end
29
- if ordered
30
- r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
31
- else
32
- r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
17
+ if concurrency < sub_jobs.count
18
+ man_batch_id = SecureRandom.urlsafe_base64(10)
19
+
20
+ Batch.redis do |r|
21
+ r.multi do
22
+ r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
23
+ r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
24
+ r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
25
+ r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
26
+
27
+ mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
28
+ j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
29
+ j = ActiveJob::Arguments.serialize([j])
30
+ JSON.unparse(j)
31
+ end
32
+ if ordered
33
+ r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
34
+ else
35
+ r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
36
+ end
37
+ r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
33
38
  end
34
- r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
35
39
  end
40
+
41
+ root_batch.allow_context_changes = (concurrency == 1)
42
+ root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
43
+
44
+ desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
36
45
  end
37
46
 
38
- root_batch.description = "Managed Batch Root (#{man_batch_id})"
39
- root_batch.allow_context_changes = (concurrency == 1)
40
47
  root_batch.context = context
41
- root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
42
- root_batch.jobs {}
43
48
 
44
- concurrency.times do
45
- self.class.perform_next_sequence_job(man_batch_id)
49
+ blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
50
+
51
+ root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
52
+
53
+ if concurrency < sub_jobs.count
54
+ root_batch.jobs {}
55
+ concurrency.times do
56
+ perform_next_sequence_job(man_batch_id)
57
+ end
58
+ else
59
+ root_batch.jobs do
60
+ sub_jobs.each do |j|
61
+ ChainBuilder.enqueue_job(j)
62
+ end
63
+ end
46
64
  end
47
65
  end
48
66
 
67
+ def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
68
+ self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
69
+ end
70
+
49
71
  def self.cleanup_redis(status, options)
50
72
  man_batch_id = options['managed_batch_id']
51
73
  Batch.redis do |r|
@@ -94,6 +116,21 @@ module CanvasSync
94
116
  end
95
117
  end
96
118
  end
119
+
120
+ class ManagedBatchProxy
121
+ def initialize(real_batch)
122
+ @real_batch = real_batch
123
+ end
124
+
125
+ delegate_missing_to :real_batch
126
+
127
+ def jobs
128
+ raise "Managed Batches do not support calling .jobs directly!"
129
+ end
130
+
131
+ private
132
+ attr_reader :real_batch
133
+ end
97
134
  end
98
135
  end
99
136
  end
@@ -3,14 +3,20 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class SerialBatchJob < BaseJob
6
- def perform(sub_jobs, context: nil)
7
- ManagedBatchJob.new.perform(
6
+ def self.make_batch(sub_jobs, context: nil, &blk)
7
+ ManagedBatchJob.make_batch(
8
8
  sub_jobs,
9
- context: context,
10
9
  ordered: true,
11
10
  concurrency: false,
11
+ context: context,
12
+ desc_prefix: 'SerialBatchJob',
13
+ &blk
12
14
  )
13
15
  end
16
+
17
+ def perform(sub_jobs, context: nil)
18
+ self.class.make_batch(sub_jobs, context: context)
19
+ end
14
20
  end
15
21
  end
16
22
  end
@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
12
12
  end
13
13
 
14
14
  def drain_zset(key)
15
- items, _ = Sidekiq.redis do |r|
15
+ items, _ = Batch.redis do |r|
16
16
  r.multi do
17
17
  r.zrange(key, 0, -1)
18
18
  r.zremrangebyrank(key, 0, -1)
@@ -8,11 +8,11 @@ module CanvasSync
8
8
  module Sidekiq
9
9
  module WorkerExtension
10
10
  def bid
11
- Thread.current[:batch].bid
11
+ Thread.current[CURRENT_BATCH_THREAD_KEY].bid
12
12
  end
13
13
 
14
14
  def batch
15
- Thread.current[:batch]
15
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
16
16
  end
17
17
 
18
18
  def batch_context
@@ -42,7 +42,7 @@ module CanvasSync
42
42
 
43
43
  class ClientMiddleware
44
44
  def call(_worker, msg, _queue, _redis_pool = nil)
45
- if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
45
+ if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
46
46
  batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
47
47
  end
48
48
  yield
@@ -57,17 +57,17 @@ module CanvasSync
57
57
  class ServerMiddleware
58
58
  def call(_worker, msg, _queue)
59
59
  if (bid = msg['bid'])
60
- prev_batch = Thread.current[:batch]
60
+ prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
61
61
  begin
62
- Thread.current[:batch] = Batch.new(bid)
62
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
63
63
  yield
64
- Thread.current[:batch].save_context_changes
64
+ Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
65
65
  Batch.process_successful_job(bid, msg['jid'])
66
66
  rescue
67
67
  Batch.process_failed_job(bid, msg['jid'])
68
68
  raise
69
69
  ensure
70
- Thread.current[:batch] = prev_batch
70
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
71
71
  end
72
72
  else
73
73
  yield
@@ -14,6 +14,12 @@ module CanvasSync
14
14
  globals[:updated_after] = last_batch&.started_at&.iso8601
15
15
  end
16
16
 
17
+ # Refuse to run syncs of the same genre if there is a running full sync
18
+ if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
19
+ Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
20
+ return
21
+ end
22
+
17
23
  if should_full_sync?(globals[:full_sync_every])
18
24
  globals[:updated_after] = nil
19
25
  end
@@ -25,6 +31,10 @@ module CanvasSync
25
31
  status: 'processing',
26
32
  )
27
33
 
34
+ globals[:batch_genre] = genre
35
+ globals[:batch_start_time] = sync_batch.started_at.iso8601
36
+ globals[:sync_batch_id] = sync_batch.id
37
+
28
38
  JobBatches::Batch.new.tap do |b|
29
39
  b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
30
40
  b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
@@ -62,7 +72,7 @@ module CanvasSync
62
72
  end
63
73
 
64
74
  def last_full_sync_record
65
- @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
75
+ @last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
66
76
  end
67
77
 
68
78
  def last_full_sync
@@ -23,23 +23,7 @@ module CanvasSync
23
23
  end
24
24
  end
25
25
 
26
- if (jobs = options[:sub_jobs]).present?
27
- context = options[:context] || {}
28
- if options[:term_scope]
29
- Term.send(options[:term_scope]).find_each.map do |term|
30
- local_context = context.merge(canvas_term_id: get_term_id(term))
31
- JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
32
- end
33
- else
34
- JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
35
- end
36
- end
37
- end
38
-
39
- protected
40
-
41
- def get_term_id(term)
42
- term.try(:canvas_id) || term.canvas_term_id
26
+ TermBatchesJob.perform_now(options)
43
27
  end
44
28
  end
45
29
  end
@@ -0,0 +1,50 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class TermBatchesJob < CanvasSync::Job
4
+ def perform(options)
5
+ if (jobs = options[:sub_jobs]).present?
6
+ context = options[:context] || {}
7
+ if options[:term_scope]
8
+ Term.send(options[:term_scope]).find_each.map do |term|
9
+ term_id = get_term_id(term)
10
+ local_context = context.merge(canvas_term_id: term_id)
11
+
12
+ # Override the delta-syncing date if:
13
+ # 1. the Term hasn't been synced before or
14
+ # 2. the Term underwent a period of not syncing
15
+ term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
16
+ if batch_context[:updated_after]
17
+ if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
18
+ local_context[:updated_after] = term_last_sync.presence
19
+ end
20
+ end
21
+
22
+ JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
23
+ b.description = "TermBatchJob(#{term.canvas_id}) Root"
24
+ b.context = local_context
25
+ b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
26
+ end
27
+ end
28
+ else
29
+ JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
30
+ end
31
+ end
32
+ end
33
+
34
+ def self.batch_finished(status, opts)
35
+ ctx = JobBatches::Batch.current_context
36
+ term_id = ctx[:canvas_term_id]
37
+ CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
38
+ end
39
+
40
+ def self.last_sync_key(term_id)
41
+ ctx = JobBatches::Batch.current_context
42
+ "#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
43
+ end
44
+
45
+ def get_term_id(term)
46
+ term.try(:canvas_id) || term.canvas_term_id
47
+ end
48
+ end
49
+ end
50
+ end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.23.beta8".freeze
2
+ VERSION = "0.17.27.beta1".freeze
3
3
  end
data/lib/canvas_sync.rb CHANGED
@@ -80,6 +80,7 @@ module CanvasSync
80
80
  ].freeze
81
81
 
82
82
  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
83
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::TermBatchesJob, :sub_jobs)
83
84
  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
84
85
 
85
86
  class << self
@@ -130,12 +131,8 @@ module CanvasSync
130
131
  models,
131
132
  term_scope: nil,
132
133
  term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
133
- legacy_support: false,
134
- account_id: nil,
135
- updated_after: nil,
136
- full_sync_every: nil,
137
- batch_genre: nil,
138
- options: {}
134
+ options: {},
135
+ **kwargs
139
136
  ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
140
137
  return unless models.present?
141
138
  models.map! &:to_s
@@ -155,7 +152,7 @@ module CanvasSync
155
152
  context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
156
153
  }.with_indifferent_access
157
154
 
158
- root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
155
+ root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
159
156
  concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
160
157
  root_chain << concurrent_root_chain
161
158
  current_chain = concurrent_root_chain
@@ -211,6 +208,20 @@ module CanvasSync
211
208
  # Wrap it all up
212
209
  ###############################
213
210
 
211
+ root_chain
212
+ end
213
+
214
+ def base_canvas_sync_chain(
215
+ legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
216
+ account_id: nil, # legacy/non PandaPal apps
217
+ updated_after: nil,
218
+ full_sync_every: nil,
219
+ batch_genre: nil,
220
+ globals: {},
221
+ &blk
222
+ )
223
+ root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
224
+
214
225
  global_options = {
215
226
  legacy_support: legacy_support,
216
227
  updated_after: updated_after,
@@ -218,10 +229,12 @@ module CanvasSync
218
229
  batch_genre: batch_genre,
219
230
  }
220
231
  global_options[:account_id] = account_id if account_id.present?
221
- global_options.merge!(options[:global]) if options[:global].present?
232
+ global_options.merge!(globals) if globals
222
233
 
223
234
  root_chain.params[1] = global_options
224
235
 
236
+ root_chain.apply_block(&blk)
237
+
225
238
  root_chain
226
239
  end
227
240
 
@@ -307,5 +320,15 @@ module CanvasSync
307
320
  return if invalid.empty?
308
321
  raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
309
322
  end
323
+
324
+ def redis(*args, &blk)
325
+ JobBatches::Batch.redis(*args, &blk)
326
+ end
327
+
328
+ def redis_prefix
329
+ pfx = "cs"
330
+ pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
331
+ pfx
332
+ end
310
333
  end
311
334
  end