canvas_sync 0.17.23 → 0.17.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1e0b64f56f9bd68aa468f8bd608000f622c07a0dbcf1a49469435788e5708545
4
- data.tar.gz: 48bd87ac7843bf65f5e4ac11bc5271a96a032244edcb2996e710006ecf007801
3
+ metadata.gz: b85b5374a289fbab25718bb03812c9623065dc5c8b4e35fff67c79d220d7fda9
4
+ data.tar.gz: 9a8d01f7a45bfe58339c8ba8a808a29d655711945ec7cf27a8e8fcd02205f343
5
5
  SHA512:
6
- metadata.gz: 37835e33df63bbca5a712535b1fa32574e7f6d80a186cdb1e1638b5232ccb8f96cf36c95ae10483e7e519a0ddfee7334f91cd323757302da0304d15205274a62
7
- data.tar.gz: 785bc9a767cad6b750c23dc16f7f5fec58e88c5fb8defe7156e7931c31ee5093de25c864aa9c4b2dd6aa72f884fbbc28fcd707dfda59a8f048d551c1a8c0d361
6
+ metadata.gz: 73bf2bd012676ff339ccb3f2c7a2802e0752bc5623ccd301fc808d15ce139673be955ae4aee2b9ee842b18c2235cdbc843f197090c00b4caf6f57c6975d5c81a
7
+ data.tar.gz: eed6e24b8f30396a2f09751d2b786527a7b039c8b6fa6fbdcc9bdf695449f6e90ca76e138aec9fefb3a905abdbefd45ca092a1e1b85b0f706f6ca0fc35ff9fde
@@ -65,7 +65,7 @@ module CanvasSync
65
65
  }
66
66
 
67
67
  row_buffer = nil
68
- if defined?(User) && klass == User && csv_column_names.include?(:user_id)
68
+ if defined?(User) && klass == User && csv_column_names.include?('user_id')
69
69
  row_buffer = UserRowBuffer.new(&row_buffer_out)
70
70
  else
71
71
  row_buffer = NullRowBuffer.new(&row_buffer_out)
@@ -19,6 +19,8 @@ require_relative "./chain_builder"
19
19
 
20
20
  module CanvasSync
21
21
  module JobBatches
22
+ CURRENT_BATCH_THREAD_KEY = :job_batches_batch
23
+
22
24
  class Batch
23
25
  include RedisModel
24
26
 
@@ -89,7 +91,7 @@ module CanvasSync
89
91
  raise NoBlockGivenError unless block_given?
90
92
 
91
93
  if !@existing && !@initialized
92
- parent_bid = Thread.current[:batch]&.bid
94
+ parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
93
95
 
94
96
  redis.multi do |r|
95
97
  r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
@@ -111,11 +113,11 @@ module CanvasSync
111
113
  end
112
114
 
113
115
  begin
114
- parent = Thread.current[:batch]
115
- Thread.current[:batch] = self
116
+ parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
117
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = self
116
118
  yield
117
119
  ensure
118
- Thread.current[:batch] = parent
120
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
119
121
  end
120
122
 
121
123
  nil
@@ -179,11 +181,11 @@ module CanvasSync
179
181
 
180
182
  def self.with_batch(batch)
181
183
  batch = self.new(batch) if batch.is_a?(String)
182
- parent = Thread.current[:batch]
183
- Thread.current[:batch] = batch
184
+ parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
185
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
184
186
  yield
185
187
  ensure
186
- Thread.current[:batch] = parent
188
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
187
189
  end
188
190
 
189
191
  # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
@@ -228,6 +230,14 @@ module CanvasSync
228
230
  end
229
231
 
230
232
  class << self
233
+ def current
234
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
235
+ end
236
+
237
+ def current_context
238
+ current&.context || {}
239
+ end
240
+
231
241
  def process_failed_job(bid, jid)
232
242
  _, pending, failed, children, complete, parent_bid = redis do |r|
233
243
  return unless r.exists?("BID-#{bid}")
@@ -6,17 +6,17 @@ module CanvasSync
6
6
  included do
7
7
  around_perform do |job, block|
8
8
  if (@bid) # This _must_ be @bid - not just bid
9
- prev_batch = Thread.current[:batch]
9
+ prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
10
10
  begin
11
- Thread.current[:batch] = Batch.new(@bid)
11
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
12
12
  block.call
13
- Thread.current[:batch].save_context_changes
13
+ Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
14
14
  Batch.process_successful_job(@bid, job_id)
15
15
  rescue
16
16
  Batch.process_failed_job(@bid, job_id)
17
17
  raise
18
18
  ensure
19
- Thread.current[:batch] = prev_batch
19
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
20
20
  end
21
21
  else
22
22
  block.call
@@ -24,7 +24,7 @@ module CanvasSync
24
24
  end
25
25
 
26
26
  around_enqueue do |job, block|
27
- if (batch = Thread.current[:batch])
27
+ if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
28
28
  @bid = batch.bid
29
29
  batch.increment_job_queue(job_id) if @bid
30
30
  end
@@ -33,11 +33,11 @@ module CanvasSync
33
33
  end
34
34
 
35
35
  def bid
36
- @bid || Thread.current[:batch]&.bid
36
+ @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
37
37
  end
38
38
 
39
39
  def batch
40
- Thread.current[:batch]
40
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
41
41
  end
42
42
 
43
43
  def batch_context
@@ -37,16 +37,29 @@ module CanvasSync
37
37
  insert_at(-1, new_job)
38
38
  end
39
39
 
40
- def insert_at(position, new_jobs)
40
+ def insert_at(position, new_jobs, *args, &blk)
41
41
  chain = self.class.get_chain_parameter(base_job)
42
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
43
+ new_jobs = build_job_hash(new_jobs, *args, &blk)
44
+ elsif args.length > 0
45
+ raise "Unexpected number of arguments"
46
+ end
42
47
  new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
43
48
  chain.insert(position, *new_jobs)
44
49
  end
45
50
 
46
- def insert(new_jobs, **kwargs)
47
- invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
48
- raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
49
- raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
51
+ def insert(new_jobs, *args, **kwargs, &blk)
52
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
53
+ job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
54
+ args << job_kwargs if job_kwargs.present?
55
+ new_jobs = build_job_hash(new_jobs, *args, &blk)
56
+ kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
57
+ else
58
+ invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
59
+ raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
60
+ raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
61
+ raise "Unexpected number of arguments" if args.length > 0
62
+ end
50
63
 
51
64
  new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
52
65
 
@@ -90,7 +103,6 @@ module CanvasSync
90
103
  raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
91
104
  return nil if matching_jobs.count == 0
92
105
 
93
-
94
106
  job = matching_jobs[0][0]
95
107
  job = self.class.new(job) unless job.is_a?(ChainBuilder)
96
108
  job
@@ -116,8 +128,22 @@ module CanvasSync
116
128
  end
117
129
  end
118
130
 
131
+ def apply_block(&blk)
132
+ return unless blk.present?
133
+ instance_exec(&blk)
134
+ end
135
+
119
136
  private
120
137
 
138
+ def build_job_hash(job, *params, &blk)
139
+ hsh = {
140
+ job: job,
141
+ parameters: params,
142
+ }
143
+ self.class.new(hsh).apply_block(&blk) if blk.present?
144
+ hsh
145
+ end
146
+
121
147
  def find_matching_jobs(search_job, parent_job = self.base_job)
122
148
  return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
123
149
 
@@ -150,6 +176,13 @@ module CanvasSync
150
176
  end
151
177
 
152
178
  class << self
179
+ def build(job, *args, &blk)
180
+ new(job).tap do |ch|
181
+ ch[:parameters] = args
182
+ ch.apply_block(&blk)
183
+ end
184
+ end
185
+
153
186
  def _job_type_definitions
154
187
  @job_type_definitions ||= {}
155
188
  end
@@ -95,6 +95,10 @@ module CanvasSync
95
95
  @flattened = flattened.with_indifferent_access
96
96
  end
97
97
 
98
+ def to_h
99
+ flatten
100
+ end
101
+
98
102
  private
99
103
 
100
104
  def get_parent_hash(bid)
@@ -3,16 +3,19 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class ConcurrentBatchJob < BaseJob
6
+ def self.make_batch(sub_jobs, context: nil, &blk)
7
+ ManagedBatchJob.make_batch(
8
+ sub_jobs,
9
+ ordered: false,
10
+ concurrency: true,
11
+ context: context,
12
+ desc_prefix: 'ConcurrentBatchJob',
13
+ &blk
14
+ )
15
+ end
16
+
6
17
  def perform(sub_jobs, context: nil)
7
- Batch.new.tap do |b|
8
- b.description = "Concurrent Batch Root"
9
- b.context = context
10
- b.jobs do
11
- sub_jobs.each do |j|
12
- ChainBuilder.enqueue_job(j)
13
- end
14
- end
15
- end
18
+ self.class.make_batch(sub_jobs, context: context)
16
19
  end
17
20
  end
18
21
  end
@@ -3,8 +3,8 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class ManagedBatchJob < BaseJob
6
- def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
7
- man_batch_id = SecureRandom.urlsafe_base64(10)
6
+ def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
7
+ desc_prefix ||= ''
8
8
 
9
9
  if concurrency == 0 || concurrency == nil || concurrency == true
10
10
  concurrency = sub_jobs.count
@@ -14,38 +14,60 @@ module CanvasSync
14
14
 
15
15
  root_batch = Batch.new
16
16
 
17
- Batch.redis do |r|
18
- r.multi do
19
- r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
20
- r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
21
- r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
22
- r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
23
-
24
- mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
25
- j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
26
- j = ActiveJob::Arguments.serialize([j])
27
- JSON.unparse(j)
28
- end
29
- if ordered
30
- r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
31
- else
32
- r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
17
+ if concurrency < sub_jobs.count
18
+ man_batch_id = SecureRandom.urlsafe_base64(10)
19
+
20
+ Batch.redis do |r|
21
+ r.multi do
22
+ r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
23
+ r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
24
+ r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
25
+ r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
26
+
27
+ mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
28
+ j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
29
+ j = ActiveJob::Arguments.serialize([j])
30
+ JSON.unparse(j)
31
+ end
32
+ if ordered
33
+ r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
34
+ else
35
+ r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
36
+ end
37
+ r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
33
38
  end
34
- r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
35
39
  end
40
+
41
+ root_batch.allow_context_changes = (concurrency == 1)
42
+ root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
43
+
44
+ desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
36
45
  end
37
46
 
38
- root_batch.description = "Managed Batch Root (#{man_batch_id})"
39
- root_batch.allow_context_changes = (concurrency == 1)
40
47
  root_batch.context = context
41
- root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
42
- root_batch.jobs {}
43
48
 
44
- concurrency.times do
45
- self.class.perform_next_sequence_job(man_batch_id)
49
+ blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
50
+
51
+ root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
52
+
53
+ if concurrency < sub_jobs.count
54
+ root_batch.jobs {}
55
+ concurrency.times do
56
+ perform_next_sequence_job(man_batch_id)
57
+ end
58
+ else
59
+ root_batch.jobs do
60
+ sub_jobs.each do |j|
61
+ ChainBuilder.enqueue_job(j)
62
+ end
63
+ end
46
64
  end
47
65
  end
48
66
 
67
+ def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
68
+ self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
69
+ end
70
+
49
71
  def self.cleanup_redis(status, options)
50
72
  man_batch_id = options['managed_batch_id']
51
73
  Batch.redis do |r|
@@ -94,6 +116,21 @@ module CanvasSync
94
116
  end
95
117
  end
96
118
  end
119
+
120
+ class ManagedBatchProxy
121
+ def initialize(real_batch)
122
+ @real_batch = real_batch
123
+ end
124
+
125
+ delegate_missing_to :real_batch
126
+
127
+ def jobs
128
+ raise "Managed Batches do not support calling .jobs directly!"
129
+ end
130
+
131
+ private
132
+ attr_reader :real_batch
133
+ end
97
134
  end
98
135
  end
99
136
  end
@@ -3,14 +3,20 @@ require_relative './base_job'
3
3
  module CanvasSync
4
4
  module JobBatches
5
5
  class SerialBatchJob < BaseJob
6
- def perform(sub_jobs, context: nil)
7
- ManagedBatchJob.new.perform(
6
+ def self.make_batch(sub_jobs, context: nil, &blk)
7
+ ManagedBatchJob.make_batch(
8
8
  sub_jobs,
9
- context: context,
10
9
  ordered: true,
11
10
  concurrency: false,
11
+ context: context,
12
+ desc_prefix: 'SerialBatchJob',
13
+ &blk
12
14
  )
13
15
  end
16
+
17
+ def perform(sub_jobs, context: nil)
18
+ self.class.make_batch(sub_jobs, context: context)
19
+ end
14
20
  end
15
21
  end
16
22
  end
@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
12
12
  end
13
13
 
14
14
  def drain_zset(key)
15
- items, _ = Sidekiq.redis do |r|
15
+ items, _ = Batch.redis do |r|
16
16
  r.multi do
17
17
  r.zrange(key, 0, -1)
18
18
  r.zremrangebyrank(key, 0, -1)
@@ -8,11 +8,11 @@ module CanvasSync
8
8
  module Sidekiq
9
9
  module WorkerExtension
10
10
  def bid
11
- Thread.current[:batch].bid
11
+ Thread.current[CURRENT_BATCH_THREAD_KEY].bid
12
12
  end
13
13
 
14
14
  def batch
15
- Thread.current[:batch]
15
+ Thread.current[CURRENT_BATCH_THREAD_KEY]
16
16
  end
17
17
 
18
18
  def batch_context
@@ -42,7 +42,7 @@ module CanvasSync
42
42
 
43
43
  class ClientMiddleware
44
44
  def call(_worker, msg, _queue, _redis_pool = nil)
45
- if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
45
+ if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
46
46
  batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
47
47
  end
48
48
  yield
@@ -57,17 +57,17 @@ module CanvasSync
57
57
  class ServerMiddleware
58
58
  def call(_worker, msg, _queue)
59
59
  if (bid = msg['bid'])
60
- prev_batch = Thread.current[:batch]
60
+ prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
61
61
  begin
62
- Thread.current[:batch] = Batch.new(bid)
62
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
63
63
  yield
64
- Thread.current[:batch].save_context_changes
64
+ Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
65
65
  Batch.process_successful_job(bid, msg['jid'])
66
66
  rescue
67
67
  Batch.process_failed_job(bid, msg['jid'])
68
68
  raise
69
69
  ensure
70
- Thread.current[:batch] = prev_batch
70
+ Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
71
71
  end
72
72
  else
73
73
  yield
@@ -14,6 +14,12 @@ module CanvasSync
14
14
  globals[:updated_after] = last_batch&.started_at&.iso8601
15
15
  end
16
16
 
17
+ # Refuse to run syncs of the same genre if there is a running full sync
18
+ if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
19
+ Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
20
+ return
21
+ end
22
+
17
23
  if should_full_sync?(globals[:full_sync_every])
18
24
  globals[:updated_after] = nil
19
25
  end
@@ -25,6 +31,10 @@ module CanvasSync
25
31
  status: 'processing',
26
32
  )
27
33
 
34
+ globals[:batch_genre] = genre
35
+ globals[:batch_start_time] = sync_batch.started_at.iso8601
36
+ globals[:sync_batch_id] = sync_batch.id
37
+
28
38
  JobBatches::Batch.new.tap do |b|
29
39
  b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
30
40
  b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
@@ -62,7 +72,7 @@ module CanvasSync
62
72
  end
63
73
 
64
74
  def last_full_sync_record
65
- @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
75
+ @last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
66
76
  end
67
77
 
68
78
  def last_full_sync
@@ -23,23 +23,7 @@ module CanvasSync
23
23
  end
24
24
  end
25
25
 
26
- if (jobs = options[:sub_jobs]).present?
27
- context = options[:context] || {}
28
- if options[:term_scope]
29
- Term.send(options[:term_scope]).find_each.map do |term|
30
- local_context = context.merge(canvas_term_id: get_term_id(term))
31
- JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
32
- end
33
- else
34
- JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
35
- end
36
- end
37
- end
38
-
39
- protected
40
-
41
- def get_term_id(term)
42
- term.try(:canvas_id) || term.canvas_term_id
26
+ TermBatchesJob.perform_now(options)
43
27
  end
44
28
  end
45
29
  end
@@ -0,0 +1,50 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class TermBatchesJob < CanvasSync::Job
4
+ def perform(options)
5
+ if (jobs = options[:sub_jobs]).present?
6
+ context = options[:context] || {}
7
+ if options[:term_scope]
8
+ Term.send(options[:term_scope]).find_each.map do |term|
9
+ term_id = get_term_id(term)
10
+ local_context = context.merge(canvas_term_id: term_id)
11
+
12
+ # Override the delta-syncing date if:
13
+ # 1. the Term hasn't been synced before or
14
+ # 2. the Term underwent a period of not syncing
15
+ term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
16
+ if batch_context[:updated_after]
17
+ if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
18
+ local_context[:updated_after] = term_last_sync.presence
19
+ end
20
+ end
21
+
22
+ JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
23
+ b.description = "TermBatchJob(#{term.canvas_id}) Root"
24
+ b.context = local_context
25
+ b.on(:success, "#{self.class.to_s}.batch_finished")
26
+ end
27
+ end
28
+ else
29
+ JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
30
+ end
31
+ end
32
+ end
33
+
34
+ def self.batch_finished(status, opts)
35
+ ctx = JobBatches::Batch.current_context
36
+ term_id = ctx[:canvas_term_id]
37
+ CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
38
+ end
39
+
40
+ def self.last_sync_key(term_id)
41
+ ctx = JobBatches::Batch.current_context
42
+ "#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
43
+ end
44
+
45
+ def get_term_id(term)
46
+ term.try(:canvas_id) || term.canvas_term_id
47
+ end
48
+ end
49
+ end
50
+ end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.23".freeze
2
+ VERSION = "0.17.27".freeze
3
3
  end
data/lib/canvas_sync.rb CHANGED
@@ -80,6 +80,7 @@ module CanvasSync
80
80
  ].freeze
81
81
 
82
82
  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
83
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::TermBatchesJob, :sub_jobs)
83
84
  JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
84
85
 
85
86
  class << self
@@ -130,12 +131,8 @@ module CanvasSync
130
131
  models,
131
132
  term_scope: nil,
132
133
  term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
133
- legacy_support: false,
134
- account_id: nil,
135
- updated_after: nil,
136
- full_sync_every: nil,
137
- batch_genre: nil,
138
- options: {}
134
+ options: {},
135
+ **kwargs
139
136
  ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
140
137
  return unless models.present?
141
138
  models.map! &:to_s
@@ -155,7 +152,7 @@ module CanvasSync
155
152
  context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
156
153
  }.with_indifferent_access
157
154
 
158
- root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
155
+ root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
159
156
  concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
160
157
  root_chain << concurrent_root_chain
161
158
  current_chain = concurrent_root_chain
@@ -211,6 +208,20 @@ module CanvasSync
211
208
  # Wrap it all up
212
209
  ###############################
213
210
 
211
+ root_chain
212
+ end
213
+
214
+ def base_canvas_sync_chain(
215
+ legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
216
+ account_id: nil, # legacy/non PandaPal apps
217
+ updated_after: nil,
218
+ full_sync_every: nil,
219
+ batch_genre: nil,
220
+ globals: {},
221
+ &blk
222
+ )
223
+ root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
224
+
214
225
  global_options = {
215
226
  legacy_support: legacy_support,
216
227
  updated_after: updated_after,
@@ -218,10 +229,12 @@ module CanvasSync
218
229
  batch_genre: batch_genre,
219
230
  }
220
231
  global_options[:account_id] = account_id if account_id.present?
221
- global_options.merge!(options[:global]) if options[:global].present?
232
+ global_options.merge!(globals) if globals
222
233
 
223
234
  root_chain.params[1] = global_options
224
235
 
236
+ root_chain.apply_block(&blk)
237
+
225
238
  root_chain
226
239
  end
227
240
 
@@ -307,5 +320,15 @@ module CanvasSync
307
320
  return if invalid.empty?
308
321
  raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
309
322
  end
323
+
324
+ def redis(*args, &blk)
325
+ JobBatches::Batch.redis(*args, &blk)
326
+ end
327
+
328
+ def redis_prefix
329
+ pfx = "cs"
330
+ pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
331
+ pfx
332
+ end
310
333
  end
311
334
  end