canvas_sync 0.16.5 → 0.17.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +49 -137
  3. data/app/models/canvas_sync/sync_batch.rb +5 -0
  4. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  5. data/lib/canvas_sync/importers/bulk_importer.rb +4 -7
  6. data/lib/canvas_sync/job.rb +4 -10
  7. data/lib/canvas_sync/job_batches/batch.rb +399 -0
  8. data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
  9. data/lib/canvas_sync/job_batches/callback.rb +153 -0
  10. data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
  11. data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
  12. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  13. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
  14. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
  15. data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
  16. data/lib/canvas_sync/job_batches/status.rb +63 -0
  17. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
  18. data/lib/canvas_sync/jobs/report_checker.rb +3 -6
  19. data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
  20. data/lib/canvas_sync/jobs/report_starter.rb +28 -20
  21. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  22. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  23. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  24. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  25. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  26. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  27. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
  28. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  29. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  30. data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
  31. data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
  32. data/lib/canvas_sync/processors/assignment_groups_processor.rb +2 -3
  33. data/lib/canvas_sync/processors/assignments_processor.rb +2 -3
  34. data/lib/canvas_sync/processors/context_module_items_processor.rb +2 -3
  35. data/lib/canvas_sync/processors/context_modules_processor.rb +2 -3
  36. data/lib/canvas_sync/processors/normal_processor.rb +1 -2
  37. data/lib/canvas_sync/processors/provisioning_report_processor.rb +2 -10
  38. data/lib/canvas_sync/processors/submissions_processor.rb +2 -3
  39. data/lib/canvas_sync/version.rb +1 -1
  40. data/lib/canvas_sync.rb +34 -97
  41. data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
  42. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  43. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  44. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  45. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  46. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  47. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  48. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  49. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  50. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  51. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
  52. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  53. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  54. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
  55. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  56. data/spec/dummy/config/environments/test.rb +2 -0
  57. data/spec/dummy/db/schema.rb +9 -1
  58. data/spec/job_batching/batch_aware_job_spec.rb +100 -0
  59. data/spec/job_batching/batch_spec.rb +363 -0
  60. data/spec/job_batching/callback_spec.rb +38 -0
  61. data/spec/job_batching/flow_spec.rb +91 -0
  62. data/spec/job_batching/integration/integration.rb +57 -0
  63. data/spec/job_batching/integration/nested.rb +88 -0
  64. data/spec/job_batching/integration/simple.rb +47 -0
  65. data/spec/job_batching/integration/workflow.rb +134 -0
  66. data/spec/job_batching/integration_helper.rb +48 -0
  67. data/spec/job_batching/sidekiq_spec.rb +124 -0
  68. data/spec/job_batching/status_spec.rb +92 -0
  69. data/spec/job_batching/support/base_job.rb +14 -0
  70. data/spec/job_batching/support/sample_callback.rb +2 -0
  71. data/spec/spec_helper.rb +10 -0
  72. metadata +91 -23
  73. data/lib/canvas_sync/job_chain.rb +0 -102
  74. data/lib/canvas_sync/jobs/fork_gather.rb +0 -74
  75. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
  76. data/spec/dummy/db/test.sqlite3 +0 -0
  77. data/spec/dummy/log/development.log +0 -1248
  78. data/spec/dummy/log/test.log +0 -43258
  79. data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +0 -3
  80. data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +0 -4
data/lib/canvas_sync.rb CHANGED
@@ -5,7 +5,6 @@ require "canvas_sync/engine"
5
5
  require "canvas_sync/misc_helper"
6
6
  require "canvas_sync/class_callback_executor"
7
7
  require "canvas_sync/job"
8
- require "canvas_sync/job_chain"
9
8
  require "canvas_sync/sidekiq_job"
10
9
  require "canvas_sync/api_syncable"
11
10
  require "canvas_sync/record"
@@ -14,6 +13,8 @@ require "canvas_sync/jobs/report_checker"
14
13
  require "canvas_sync/jobs/report_processor_job"
15
14
  require "canvas_sync/config"
16
15
 
16
+ require "canvas_sync/job_batches/batch"
17
+
17
18
  Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
18
19
  Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
19
20
  Dir[File.dirname(__FILE__) + "/canvas_sync/importers/*.rb"].each { |file| require file }
@@ -58,6 +59,9 @@ module CanvasSync
58
59
  graded_submissions
59
60
  ].freeze
60
61
 
62
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
63
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
64
+
61
65
  class << self
62
66
  # Runs a standard provisioning sync job with no extra report types.
63
67
  # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -73,51 +77,9 @@ module CanvasSync
73
77
  # and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
74
78
  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
75
79
  # canvas_sync_client methods require an account ID.
76
- def provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil)
80
+ def provisioning_sync(models, **kwargs)
77
81
  validate_models!(models)
78
- invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
79
- end
80
-
81
- # Runs a report different from provisioning sync job with no extra report types.
82
- #
83
- # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
84
- # want to import:
85
- # [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
86
- # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
87
- # The provisioning report will be run for each of the terms contained in that scope.
88
- # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
89
- # canvas_sync_client methods require an account ID.
90
- def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
91
- invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
92
- end
93
-
94
- # Runs a chain of ordered jobs
95
- #
96
- # See the README for usage and examples
97
- #
98
- # @param job_chain [Hash]
99
- def process_jobs(job_chain)
100
- invoke_next(job_chain)
101
- end
102
-
103
- # @deprecated
104
- def duplicate_chain(job_chain)
105
- Marshal.load(Marshal.dump(job_chain))
106
- end
107
-
108
- # Invokes the next job in a chain of jobs.
109
- #
110
- # This should typically be called automatically by the gem where necessary.
111
- #
112
- # @param job_chain [Hash] A chain of jobs to execute
113
- def invoke_next(job_chain, extra_options: {})
114
- job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
115
- job_chain.perform_next(extra_options)
116
- end
117
-
118
- def fork(job_log, job_chain, keys: [], &blk)
119
- job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
120
- job_chain.fork(job_log, keys: keys, &blk)
82
+ default_provisioning_report_chain(models, **kwargs).process!
121
83
  end
122
84
 
123
85
  # Given a Model or Relation, scope it down to items that should be synced
@@ -134,33 +96,6 @@ module CanvasSync
134
96
  scope
135
97
  end
136
98
 
137
- # Syn any report to an specific set of models
138
- #
139
- # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
140
- # @param term_scope [String]
141
- # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
142
- # canvas_sync_client methods require an account ID.
143
- def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
144
- jobs = reports_mapping.map do |report|
145
- {
146
- job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
147
- options: {
148
- report_name: report[:report_name],
149
- model: report[:model],
150
- mapping: report[:model],
151
- klass: report[:model].singularize.capitalize.to_s,
152
- term_scope: term_scope,
153
- params: report[:params]
154
- }
155
- }
156
- end
157
-
158
- global_options = {}
159
- global_options[:account_id] = account_id if account_id.present?
160
-
161
- JobChain.new(jobs: jobs, global_options: global_options)
162
- end
163
-
164
99
  # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
165
100
  #
166
101
  # @param models [Array<String>]
@@ -171,7 +106,7 @@ module CanvasSync
171
106
  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
172
107
  # canvas_sync_client methods require an account ID.
173
108
  # @return [Hash]
174
- def default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
109
+ def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
175
110
  return unless models.present?
176
111
  models.map! &:to_s
177
112
  term_scope = term_scope.to_s if term_scope
@@ -190,64 +125,66 @@ module CanvasSync
190
125
  context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
191
126
  }.with_indifferent_access
192
127
 
193
- jobs = []
128
+ root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
129
+ concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
130
+ root_chain << concurrent_root_chain
131
+ current_chain = concurrent_root_chain
132
+
194
133
  try_add_model_job = ->(model) {
195
134
  return unless models.include?(model)
196
- jobs.push(job: model_job_map[model].to_s, options: options[model.to_sym] || {})
135
+ current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
197
136
  models -= [model]
198
137
  }
199
138
 
200
139
  ##############################
201
- # Pre provisioning report jobs
140
+ # General provisioning jobs (not term-scoped)
202
141
  ##############################
203
142
 
204
- # Always sync Terms first
205
- models.unshift('terms') unless models.include?('terms')
206
- try_add_model_job.call('terms')
143
+ # Terms are always synced regardless of the models option
144
+ models -= ['terms']
207
145
 
208
- # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
146
+ # Accounts, users, roles, and admins cannot be scoped to term
209
147
  try_add_model_job.call('accounts')
210
148
 
211
149
  # These Models use the provisioning report, but are not term-scoped,
212
- # so we sync them before to ensure work is not duplicated
150
+ # so we sync them outside of the term scoping to ensure work is not duplicated
213
151
  if term_scope.present?
214
152
  models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
215
- jobs.concat(
216
- generate_provisioning_jobs(first_provisioning_models, options)
217
- )
153
+ current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
218
154
  end
219
155
 
220
156
  try_add_model_job.call('roles')
221
157
  try_add_model_job.call('admins')
222
- pre_provisioning_jobs = jobs
223
158
 
224
159
  ###############################
225
- # Post provisioning report jobs
160
+ # Per-term provisioning jobs
226
161
  ###############################
227
162
 
228
- jobs = []
163
+ per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
164
+ current_chain << per_term_chain
165
+ current_chain = per_term_chain
166
+
229
167
  try_add_model_job.call('assignments')
230
168
  try_add_model_job.call('submissions')
231
169
  try_add_model_job.call('assignment_groups')
232
170
  try_add_model_job.call('context_modules')
233
171
  try_add_model_job.call('context_module_items')
234
- post_provisioning_jobs = jobs
172
+
173
+ current_chain.insert(
174
+ generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users'])
175
+ )
235
176
 
236
177
  ###############################
237
- # Main provisioning job and queueing
178
+ # Wrap it all up
238
179
  ###############################
239
180
 
240
- jobs = [
241
- *pre_provisioning_jobs,
242
- *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
243
- *post_provisioning_jobs,
244
- ]
245
-
246
- global_options = { legacy_support: legacy_support }
181
+ global_options = { legacy_support: legacy_support, updated_after: updated_after }
247
182
  global_options[:account_id] = account_id if account_id.present?
248
183
  global_options.merge!(options[:global]) if options[:global].present?
249
184
 
250
- JobChain.new(jobs: jobs, global_options: global_options)
185
+ root_chain.params[1] = global_options
186
+
187
+ root_chain
251
188
  end
252
189
 
253
190
  def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
@@ -3,13 +3,12 @@ require 'spec_helper'
3
3
  RSpec.describe CanvasSync do
4
4
  describe '.provisioning_sync' do
5
5
  it 'invokes the first job in the queue and passes on the rest of the job chain' do
6
- expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil).chain_data
7
- first_job = expected_job_chain[:jobs].shift
6
+ expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:parameters][0]
8
7
 
9
- expect(CanvasSync::Jobs::SyncTermsJob).to receive(:perform_later)
8
+ expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
10
9
  .with(
11
10
  expected_job_chain,
12
- first_job[:options]
11
+ anything
13
12
  )
14
13
 
15
14
  CanvasSync.provisioning_sync(['courses'])
@@ -25,32 +24,42 @@ RSpec.describe CanvasSync do
25
24
 
26
25
  describe '.default_provisioning_report_chain' do
27
26
  it 'splits an options: Hash into options for separate models' do
28
- chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active, options: {
27
+ chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active, options: {
29
28
  terms: { a: 1 },
30
29
  users: { b: 2 },
31
30
  provisioning: { c: 3 },
32
31
  global: { d: 4 },
33
32
  })
34
- expect(chain.chain_data).to eq({
35
- jobs: [
36
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
37
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
38
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
39
- ],
40
- global_options: { legacy_support: false, d: 4 }
33
+ expect(chain.normalize!).to eq({
34
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
35
+ :parameters => [[
36
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
37
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
38
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
39
+ :sub_jobs=>[
40
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active", :c=>3}}
41
+ ]
42
+ }]}
43
+ ]]}
44
+ ], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
41
45
  })
42
46
  end
43
47
 
44
48
  context 'we are syncing users with a term scope' do
45
49
  it 'syncs the users in a separate job that runs first' do
46
- chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active)
47
- expect(chain.chain_data).to eq({
48
- jobs: [
49
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
50
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
51
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
52
- ],
53
- global_options: { legacy_support: false }
50
+ chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
51
+ expect(chain.normalize!).to eq({
52
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
53
+ :parameters => [[
54
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
55
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
56
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
57
+ :sub_jobs=>[
58
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
59
+ ]
60
+ }]}
61
+ ]]}
62
+ ], {:legacy_support=>false, :updated_after=>nil}]
54
63
  })
55
64
  end
56
65
  end
@@ -58,26 +67,36 @@ RSpec.describe CanvasSync do
58
67
  context 'we are syncing users without a term scope' do
59
68
  it 'syncs users along with the rest of the provisioning report' do
60
69
  chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
61
- expect(chain.chain_data).to eq({
62
- jobs: [
63
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
64
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['users', 'courses'] } }
65
- ],
66
- global_options: { legacy_support: false }
70
+ expect(chain.normalize!).to eq({
71
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
72
+ :parameters => [[
73
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
74
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
75
+ :sub_jobs=>[
76
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"], :term_scope=>nil}}
77
+ ]
78
+ }]}
79
+ ]]}
80
+ ], {:legacy_support=>false, :updated_after=>nil}],
67
81
  })
68
82
  end
69
83
  end
70
84
 
71
85
  context 'we are syncing roles with a term scope' do
72
86
  it 'syncs the roles in a separate job that runs first' do
73
- chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], :active)
74
- expect(chain.chain_data).to eq({
75
- jobs: [
76
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
77
- { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
78
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
79
- ],
80
- global_options: { legacy_support: false }
87
+ chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
88
+ expect(chain.normalize!).to eq({
89
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
90
+ :parameters => [[
91
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
92
+ {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
93
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
94
+ :sub_jobs=>[
95
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
96
+ ]
97
+ }]}
98
+ ]]}
99
+ ], {:legacy_support=>false, :updated_after=>nil}],
81
100
  })
82
101
  end
83
102
  end
@@ -85,27 +104,37 @@ RSpec.describe CanvasSync do
85
104
  context 'we are syncing roles without a term scope' do
86
105
  it 'syncs roles separately even with no term scope' do
87
106
  chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
88
- expect(chain.chain_data).to eq({
89
- jobs: [
90
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
91
- { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
92
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['courses'] } }
93
- ],
94
- global_options: { legacy_support: false }
107
+ expect(chain.normalize!).to eq({
108
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
109
+ :parameters => [[
110
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
111
+ {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
112
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
113
+ :sub_jobs=>[
114
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
115
+ ]
116
+ }]}
117
+ ]]}
118
+ ], {:legacy_support=>false, :updated_after=>nil}],
95
119
  })
96
120
  end
97
121
  end
98
122
 
99
123
  context 'we are syncing admins with a term scope' do
100
124
  it 'syncs the admins in a separate job that runs first' do
101
- chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], :active)
102
- expect(chain.chain_data).to eq({
103
- jobs: [
104
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
105
- { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
106
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
107
- ],
108
- global_options: { legacy_support: false }
125
+ chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
126
+ expect(chain.normalize!).to eq({
127
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
128
+ :parameters => [[
129
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
130
+ {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
131
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
132
+ :sub_jobs=>[
133
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
134
+ ]
135
+ }]}
136
+ ]]}
137
+ ], {:legacy_support=>false, :updated_after=>nil}],
109
138
  })
110
139
  end
111
140
  end
@@ -113,13 +142,18 @@ RSpec.describe CanvasSync do
113
142
  context 'we are syncing admins without a term scope' do
114
143
  it 'syncs admins separately even with no term scope' do
115
144
  chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
116
- expect(chain.chain_data).to eq({
117
- jobs: [
118
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
119
- { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
120
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['courses'] } }
121
- ],
122
- global_options: { legacy_support: false }
145
+ expect(chain.normalize!).to eq({
146
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
147
+ :parameters => [[
148
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
149
+ {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
150
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
151
+ :sub_jobs=>[
152
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
153
+ ]
154
+ }]}
155
+ ]]}
156
+ ], {:legacy_support=>false, :updated_after=>nil}],
123
157
  })
124
158
  end
125
159
  end
@@ -128,13 +162,18 @@ RSpec.describe CanvasSync do
128
162
  it "appends the SyncAssignmentsJob" do
129
163
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
130
164
 
131
- expect(chain.chain_data).to eq(
132
- jobs: [
133
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
134
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
135
- { job: CanvasSync::Jobs::SyncAssignmentsJob.to_s, options: {} },
136
- ],
137
- global_options: { legacy_support: false },
165
+ expect(chain.normalize!).to eq(
166
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
167
+ :parameters => [[
168
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
169
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
170
+ :sub_jobs=>[
171
+ {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
172
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
173
+ ]
174
+ }]}
175
+ ]]}
176
+ ], {:legacy_support=>false, :updated_after=>nil}],
138
177
  )
139
178
  end
140
179
  end
@@ -143,13 +182,18 @@ RSpec.describe CanvasSync do
143
182
  it "appends the SyncSubmissionsJob" do
144
183
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
145
184
 
146
- expect(chain.chain_data).to eq(
147
- jobs: [
148
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
149
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
150
- { job: CanvasSync::Jobs::SyncSubmissionsJob.to_s, options: {} },
151
- ],
152
- global_options: { legacy_support: false },
185
+ expect(chain.normalize!).to eq(
186
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
187
+ :parameters => [[
188
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
189
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
190
+ :sub_jobs=>[
191
+ {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
192
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
193
+ ]
194
+ }]}
195
+ ]]}
196
+ ], {:legacy_support=>false, :updated_after=>nil}],
153
197
  )
154
198
  end
155
199
  end
@@ -158,95 +202,24 @@ RSpec.describe CanvasSync do
158
202
  it "appends the SyncAssignmentGroupsJob" do
159
203
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
160
204
 
161
- expect(chain.chain_data).to eq(
162
- jobs: [
163
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
164
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
165
- { job: CanvasSync::Jobs::SyncAssignmentGroupsJob.to_s, options: {} },
166
- ],
167
- global_options: { legacy_support: false },
205
+ expect(chain.normalize!).to eq(
206
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
207
+ :parameters => [[
208
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
209
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
210
+ :sub_jobs=>[
211
+ {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
212
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
213
+ ]
214
+ }]}
215
+ ]]}
216
+ ], {:legacy_support=>false, :updated_after=>nil}],
168
217
  )
169
218
  end
170
219
  end
171
220
  end
172
221
  end
173
222
 
174
- describe ".simple_report_sync" do
175
-
176
- it 'invokes the first job in the queue and passes on the rest of the job chain' do
177
- expected_job_chain = CanvasSync.simple_report_chain(
178
- [
179
- {
180
- report_name: 'proservices_provisioning_csv',
181
- model: 'users',
182
- params: {
183
- "parameters[include_deleted]" => true,
184
- "parameters[users]" => true
185
- }
186
- }
187
- ]
188
- ).chain_data
189
- first_job = expected_job_chain[:jobs].shift
190
-
191
- expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
192
- .with(
193
- expected_job_chain,
194
- first_job[:options]
195
- )
196
-
197
- CanvasSync.simple_report_sync(
198
- [
199
- {
200
- report_name: 'proservices_provisioning_csv',
201
- model: 'users',
202
- params: {
203
- "parameters[include_deleted]" => true,
204
- "parameters[users]" => true
205
- }
206
- }
207
- ]
208
- )
209
- end
210
-
211
- it 'receives the job chain for the specified table' do
212
- chain = CanvasSync.simple_report_chain(
213
- [
214
- {
215
- report_name: 'proservices_provisioning_csv',
216
- model: 'users',
217
- params: {
218
- "parameters[include_deleted]" => true,
219
- "parameters[users]" => true
220
- }
221
- }
222
- ]
223
- )
224
-
225
- expected_job_chain = {
226
- jobs: [
227
- {
228
- job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
229
- options: {
230
- report_name: 'proservices_provisioning_csv',
231
- model: 'users',
232
- mapping: 'users',
233
- klass: 'User',
234
- term_scope: nil,
235
- params: {
236
- "parameters[include_deleted]" => true,
237
- "parameters[users]" => true
238
- }
239
- }
240
- }
241
- ],
242
- global_options: {}
243
- }
244
-
245
- expect(chain.chain_data).to eq(expected_job_chain)
246
-
247
- end
248
- end
249
-
250
223
  describe ".sync_scope" do
251
224
 
252
225
  end
@@ -1,14 +1,14 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  class GoodJob < CanvasSync::Job
4
- def perform(job_chain, argument)
4
+ def perform(argument)
5
5
  end
6
6
  end
7
7
 
8
8
  class EvilError < StandardError; end;
9
9
 
10
10
  class BadJob < CanvasSync::Job
11
- def perform(job_chain, argument)
11
+ def perform(argument)
12
12
  raise EvilError
13
13
  end
14
14
 
@@ -25,19 +25,19 @@ RSpec.describe CanvasSync::Job do
25
25
  describe '#perform' do
26
26
  it 'creates a CanvasSync::JobLog and logs relevant data on it' do
27
27
  expect {
28
- GoodJob.perform_now({}, "argument")
28
+ GoodJob.perform_now("argument")
29
29
  }.to change { CanvasSync::JobLog.count }.by(1)
30
30
 
31
31
  job_log = CanvasSync::JobLog.last
32
32
  expect(job_log.started_at).to_not be_nil
33
33
  expect(job_log.job_class).to eq(GoodJob.to_s)
34
- expect(job_log.job_arguments).to eq([{}, "argument"])
34
+ expect(job_log.job_arguments).to eq(["argument"])
35
35
  expect(job_log.completed_at).to_not be_nil
36
36
  end
37
37
 
38
38
  it 'logs exceptions on the CanvasSync::JobLog and then re-raises' do
39
39
  expect {
40
- BadJob.perform_now({}, "argument")
40
+ BadJob.perform_now("argument")
41
41
  }.to raise_exception(StandardError)
42
42
 
43
43
  job_log = CanvasSync::JobLog.last
@@ -47,25 +47,17 @@ RSpec.describe CanvasSync::Job do
47
47
 
48
48
  it 'invokes an error handler' do
49
49
  expect(BadJob).to receive(:good_catcher).once
50
+ set_batch_context(on_failure: 'BadJob.good_catcher')
50
51
  expect {
51
- BadJob.perform_now({
52
- jobs: [],
53
- global_options: {
54
- on_failure: 'BadJob.good_catcher'
55
- }
56
- })
52
+ BadJob.perform_now()
57
53
  }.to raise_exception(StandardError)
58
54
  end
59
55
 
60
56
  it 'logs a failing error handler' do
61
57
  expect(BadJob).to receive(:bad_catcher).once.and_call_original
58
+ set_batch_context(on_failure: 'BadJob.bad_catcher')
62
59
  expect {
63
- BadJob.perform_now({
64
- jobs: [],
65
- global_options: {
66
- on_failure: 'BadJob.bad_catcher'
67
- }
68
- })
60
+ BadJob.perform_now()
69
61
  }.to raise_exception(StandardError)
70
62
  job_log = CanvasSync::JobLog.last
71
63
  expect(job_log.backtrace).to include "Error Occurred while handling an Error"