canvas_sync 0.16.5 → 0.17.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/README.md +49 -137
  3. data/app/models/canvas_sync/sync_batch.rb +5 -0
  4. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  5. data/lib/canvas_sync/importers/bulk_importer.rb +4 -7
  6. data/lib/canvas_sync/job.rb +4 -10
  7. data/lib/canvas_sync/job_batches/batch.rb +399 -0
  8. data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
  9. data/lib/canvas_sync/job_batches/callback.rb +153 -0
  10. data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
  11. data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
  12. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  13. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
  14. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
  15. data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
  16. data/lib/canvas_sync/job_batches/status.rb +63 -0
  17. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
  18. data/lib/canvas_sync/jobs/report_checker.rb +3 -6
  19. data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
  20. data/lib/canvas_sync/jobs/report_starter.rb +28 -20
  21. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  22. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  23. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  24. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  25. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  26. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  27. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
  28. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  29. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  30. data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
  31. data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
  32. data/lib/canvas_sync/processors/assignment_groups_processor.rb +2 -3
  33. data/lib/canvas_sync/processors/assignments_processor.rb +2 -3
  34. data/lib/canvas_sync/processors/context_module_items_processor.rb +2 -3
  35. data/lib/canvas_sync/processors/context_modules_processor.rb +2 -3
  36. data/lib/canvas_sync/processors/normal_processor.rb +1 -2
  37. data/lib/canvas_sync/processors/provisioning_report_processor.rb +2 -10
  38. data/lib/canvas_sync/processors/submissions_processor.rb +2 -3
  39. data/lib/canvas_sync/version.rb +1 -1
  40. data/lib/canvas_sync.rb +34 -97
  41. data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
  42. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  43. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  44. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  45. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  46. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  47. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  48. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  49. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  50. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  51. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
  52. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  53. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  54. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
  55. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  56. data/spec/dummy/config/environments/test.rb +2 -0
  57. data/spec/dummy/db/schema.rb +9 -1
  58. data/spec/job_batching/batch_aware_job_spec.rb +100 -0
  59. data/spec/job_batching/batch_spec.rb +363 -0
  60. data/spec/job_batching/callback_spec.rb +38 -0
  61. data/spec/job_batching/flow_spec.rb +91 -0
  62. data/spec/job_batching/integration/integration.rb +57 -0
  63. data/spec/job_batching/integration/nested.rb +88 -0
  64. data/spec/job_batching/integration/simple.rb +47 -0
  65. data/spec/job_batching/integration/workflow.rb +134 -0
  66. data/spec/job_batching/integration_helper.rb +48 -0
  67. data/spec/job_batching/sidekiq_spec.rb +124 -0
  68. data/spec/job_batching/status_spec.rb +92 -0
  69. data/spec/job_batching/support/base_job.rb +14 -0
  70. data/spec/job_batching/support/sample_callback.rb +2 -0
  71. data/spec/spec_helper.rb +10 -0
  72. metadata +91 -23
  73. data/lib/canvas_sync/job_chain.rb +0 -102
  74. data/lib/canvas_sync/jobs/fork_gather.rb +0 -74
  75. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
  76. data/spec/dummy/db/test.sqlite3 +0 -0
  77. data/spec/dummy/log/development.log +0 -1248
  78. data/spec/dummy/log/test.log +0 -43258
  79. data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +0 -3
  80. data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +0 -4
data/lib/canvas_sync.rb CHANGED
@@ -5,7 +5,6 @@ require "canvas_sync/engine"
5
5
  require "canvas_sync/misc_helper"
6
6
  require "canvas_sync/class_callback_executor"
7
7
  require "canvas_sync/job"
8
- require "canvas_sync/job_chain"
9
8
  require "canvas_sync/sidekiq_job"
10
9
  require "canvas_sync/api_syncable"
11
10
  require "canvas_sync/record"
@@ -14,6 +13,8 @@ require "canvas_sync/jobs/report_checker"
14
13
  require "canvas_sync/jobs/report_processor_job"
15
14
  require "canvas_sync/config"
16
15
 
16
+ require "canvas_sync/job_batches/batch"
17
+
17
18
  Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
18
19
  Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
19
20
  Dir[File.dirname(__FILE__) + "/canvas_sync/importers/*.rb"].each { |file| require file }
@@ -58,6 +59,9 @@ module CanvasSync
58
59
  graded_submissions
59
60
  ].freeze
60
61
 
62
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
63
+ JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
64
+
61
65
  class << self
62
66
  # Runs a standard provisioning sync job with no extra report types.
63
67
  # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -73,51 +77,9 @@ module CanvasSync
73
77
  # and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
74
78
  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
75
79
  # canvas_sync_client methods require an account ID.
76
- def provisioning_sync(models, term_scope: nil, legacy_support: false, account_id: nil)
80
+ def provisioning_sync(models, **kwargs)
77
81
  validate_models!(models)
78
- invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
79
- end
80
-
81
- # Runs a report different from provisioning sync job with no extra report types.
82
- #
83
- # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
84
- # want to import:
85
- # [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
86
- # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
87
- # The provisioning report will be run for each of the terms contained in that scope.
88
- # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
89
- # canvas_sync_client methods require an account ID.
90
- def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
91
- invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
92
- end
93
-
94
- # Runs a chain of ordered jobs
95
- #
96
- # See the README for usage and examples
97
- #
98
- # @param job_chain [Hash]
99
- def process_jobs(job_chain)
100
- invoke_next(job_chain)
101
- end
102
-
103
- # @deprecated
104
- def duplicate_chain(job_chain)
105
- Marshal.load(Marshal.dump(job_chain))
106
- end
107
-
108
- # Invokes the next job in a chain of jobs.
109
- #
110
- # This should typically be called automatically by the gem where necessary.
111
- #
112
- # @param job_chain [Hash] A chain of jobs to execute
113
- def invoke_next(job_chain, extra_options: {})
114
- job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
115
- job_chain.perform_next(extra_options)
116
- end
117
-
118
- def fork(job_log, job_chain, keys: [], &blk)
119
- job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
120
- job_chain.fork(job_log, keys: keys, &blk)
82
+ default_provisioning_report_chain(models, **kwargs).process!
121
83
  end
122
84
 
123
85
  # Given a Model or Relation, scope it down to items that should be synced
@@ -134,33 +96,6 @@ module CanvasSync
134
96
  scope
135
97
  end
136
98
 
137
- # Syn any report to an specific set of models
138
- #
139
- # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
140
- # @param term_scope [String]
141
- # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
142
- # canvas_sync_client methods require an account ID.
143
- def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
144
- jobs = reports_mapping.map do |report|
145
- {
146
- job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
147
- options: {
148
- report_name: report[:report_name],
149
- model: report[:model],
150
- mapping: report[:model],
151
- klass: report[:model].singularize.capitalize.to_s,
152
- term_scope: term_scope,
153
- params: report[:params]
154
- }
155
- }
156
- end
157
-
158
- global_options = {}
159
- global_options[:account_id] = account_id if account_id.present?
160
-
161
- JobChain.new(jobs: jobs, global_options: global_options)
162
- end
163
-
164
99
  # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
165
100
  #
166
101
  # @param models [Array<String>]
@@ -171,7 +106,7 @@ module CanvasSync
171
106
  # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
172
107
  # canvas_sync_client methods require an account ID.
173
108
  # @return [Hash]
174
- def default_provisioning_report_chain(models, term_scope=nil, legacy_support=false, account_id=nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
109
+ def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
175
110
  return unless models.present?
176
111
  models.map! &:to_s
177
112
  term_scope = term_scope.to_s if term_scope
@@ -190,64 +125,66 @@ module CanvasSync
190
125
  context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
191
126
  }.with_indifferent_access
192
127
 
193
- jobs = []
128
+ root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
129
+ concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
130
+ root_chain << concurrent_root_chain
131
+ current_chain = concurrent_root_chain
132
+
194
133
  try_add_model_job = ->(model) {
195
134
  return unless models.include?(model)
196
- jobs.push(job: model_job_map[model].to_s, options: options[model.to_sym] || {})
135
+ current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
197
136
  models -= [model]
198
137
  }
199
138
 
200
139
  ##############################
201
- # Pre provisioning report jobs
140
+ # General provisioning jobs (not term-scoped)
202
141
  ##############################
203
142
 
204
- # Always sync Terms first
205
- models.unshift('terms') unless models.include?('terms')
206
- try_add_model_job.call('terms')
143
+ # Terms are always synced regardless of the models option
144
+ models -= ['terms']
207
145
 
208
- # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
146
+ # Accounts, users, roles, and admins cannot be scoped to term
209
147
  try_add_model_job.call('accounts')
210
148
 
211
149
  # These Models use the provisioning report, but are not term-scoped,
212
- # so we sync them before to ensure work is not duplicated
150
+ # so we sync them outside of the term scoping to ensure work is not duplicated
213
151
  if term_scope.present?
214
152
  models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
215
- jobs.concat(
216
- generate_provisioning_jobs(first_provisioning_models, options)
217
- )
153
+ current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
218
154
  end
219
155
 
220
156
  try_add_model_job.call('roles')
221
157
  try_add_model_job.call('admins')
222
- pre_provisioning_jobs = jobs
223
158
 
224
159
  ###############################
225
- # Post provisioning report jobs
160
+ # Per-term provisioning jobs
226
161
  ###############################
227
162
 
228
- jobs = []
163
+ per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
164
+ current_chain << per_term_chain
165
+ current_chain = per_term_chain
166
+
229
167
  try_add_model_job.call('assignments')
230
168
  try_add_model_job.call('submissions')
231
169
  try_add_model_job.call('assignment_groups')
232
170
  try_add_model_job.call('context_modules')
233
171
  try_add_model_job.call('context_module_items')
234
- post_provisioning_jobs = jobs
172
+
173
+ current_chain.insert(
174
+ generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users'])
175
+ )
235
176
 
236
177
  ###############################
237
- # Main provisioning job and queueing
178
+ # Wrap it all up
238
179
  ###############################
239
180
 
240
- jobs = [
241
- *pre_provisioning_jobs,
242
- *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
243
- *post_provisioning_jobs,
244
- ]
245
-
246
- global_options = { legacy_support: legacy_support }
181
+ global_options = { legacy_support: legacy_support, updated_after: updated_after }
247
182
  global_options[:account_id] = account_id if account_id.present?
248
183
  global_options.merge!(options[:global]) if options[:global].present?
249
184
 
250
- JobChain.new(jobs: jobs, global_options: global_options)
185
+ root_chain.params[1] = global_options
186
+
187
+ root_chain
251
188
  end
252
189
 
253
190
  def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
@@ -3,13 +3,12 @@ require 'spec_helper'
3
3
  RSpec.describe CanvasSync do
4
4
  describe '.provisioning_sync' do
5
5
  it 'invokes the first job in the queue and passes on the rest of the job chain' do
6
- expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil).chain_data
7
- first_job = expected_job_chain[:jobs].shift
6
+ expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:parameters][0]
8
7
 
9
- expect(CanvasSync::Jobs::SyncTermsJob).to receive(:perform_later)
8
+ expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
10
9
  .with(
11
10
  expected_job_chain,
12
- first_job[:options]
11
+ anything
13
12
  )
14
13
 
15
14
  CanvasSync.provisioning_sync(['courses'])
@@ -25,32 +24,42 @@ RSpec.describe CanvasSync do
25
24
 
26
25
  describe '.default_provisioning_report_chain' do
27
26
  it 'splits an options: Hash into options for separate models' do
28
- chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active, options: {
27
+ chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active, options: {
29
28
  terms: { a: 1 },
30
29
  users: { b: 2 },
31
30
  provisioning: { c: 3 },
32
31
  global: { d: 4 },
33
32
  })
34
- expect(chain.chain_data).to eq({
35
- jobs: [
36
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
37
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
38
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
39
- ],
40
- global_options: { legacy_support: false, d: 4 }
33
+ expect(chain.normalize!).to eq({
34
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
35
+ :parameters => [[
36
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
37
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
38
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
39
+ :sub_jobs=>[
40
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active", :c=>3}}
41
+ ]
42
+ }]}
43
+ ]]}
44
+ ], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
41
45
  })
42
46
  end
43
47
 
44
48
  context 'we are syncing users with a term scope' do
45
49
  it 'syncs the users in a separate job that runs first' do
46
- chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active)
47
- expect(chain.chain_data).to eq({
48
- jobs: [
49
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
50
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
51
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
52
- ],
53
- global_options: { legacy_support: false }
50
+ chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
51
+ expect(chain.normalize!).to eq({
52
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
53
+ :parameters => [[
54
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
55
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
56
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
57
+ :sub_jobs=>[
58
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
59
+ ]
60
+ }]}
61
+ ]]}
62
+ ], {:legacy_support=>false, :updated_after=>nil}]
54
63
  })
55
64
  end
56
65
  end
@@ -58,26 +67,36 @@ RSpec.describe CanvasSync do
58
67
  context 'we are syncing users without a term scope' do
59
68
  it 'syncs users along with the rest of the provisioning report' do
60
69
  chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
61
- expect(chain.chain_data).to eq({
62
- jobs: [
63
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
64
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['users', 'courses'] } }
65
- ],
66
- global_options: { legacy_support: false }
70
+ expect(chain.normalize!).to eq({
71
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
72
+ :parameters => [[
73
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
74
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
75
+ :sub_jobs=>[
76
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"], :term_scope=>nil}}
77
+ ]
78
+ }]}
79
+ ]]}
80
+ ], {:legacy_support=>false, :updated_after=>nil}],
67
81
  })
68
82
  end
69
83
  end
70
84
 
71
85
  context 'we are syncing roles with a term scope' do
72
86
  it 'syncs the roles in a separate job that runs first' do
73
- chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], :active)
74
- expect(chain.chain_data).to eq({
75
- jobs: [
76
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
77
- { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
78
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
79
- ],
80
- global_options: { legacy_support: false }
87
+ chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
88
+ expect(chain.normalize!).to eq({
89
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
90
+ :parameters => [[
91
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
92
+ {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
93
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
94
+ :sub_jobs=>[
95
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
96
+ ]
97
+ }]}
98
+ ]]}
99
+ ], {:legacy_support=>false, :updated_after=>nil}],
81
100
  })
82
101
  end
83
102
  end
@@ -85,27 +104,37 @@ RSpec.describe CanvasSync do
85
104
  context 'we are syncing roles without a term scope' do
86
105
  it 'syncs roles separately even with no term scope' do
87
106
  chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
88
- expect(chain.chain_data).to eq({
89
- jobs: [
90
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
91
- { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
92
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['courses'] } }
93
- ],
94
- global_options: { legacy_support: false }
107
+ expect(chain.normalize!).to eq({
108
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
109
+ :parameters => [[
110
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
111
+ {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
112
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
113
+ :sub_jobs=>[
114
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
115
+ ]
116
+ }]}
117
+ ]]}
118
+ ], {:legacy_support=>false, :updated_after=>nil}],
95
119
  })
96
120
  end
97
121
  end
98
122
 
99
123
  context 'we are syncing admins with a term scope' do
100
124
  it 'syncs the admins in a separate job that runs first' do
101
- chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], :active)
102
- expect(chain.chain_data).to eq({
103
- jobs: [
104
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
105
- { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
106
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
107
- ],
108
- global_options: { legacy_support: false }
125
+ chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
126
+ expect(chain.normalize!).to eq({
127
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
128
+ :parameters => [[
129
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
130
+ {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
131
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
132
+ :sub_jobs=>[
133
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
134
+ ]
135
+ }]}
136
+ ]]}
137
+ ], {:legacy_support=>false, :updated_after=>nil}],
109
138
  })
110
139
  end
111
140
  end
@@ -113,13 +142,18 @@ RSpec.describe CanvasSync do
113
142
  context 'we are syncing admins without a term scope' do
114
143
  it 'syncs admins separately even with no term scope' do
115
144
  chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
116
- expect(chain.chain_data).to eq({
117
- jobs: [
118
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
119
- { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
120
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['courses'] } }
121
- ],
122
- global_options: { legacy_support: false }
145
+ expect(chain.normalize!).to eq({
146
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
147
+ :parameters => [[
148
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
149
+ {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
150
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
151
+ :sub_jobs=>[
152
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
153
+ ]
154
+ }]}
155
+ ]]}
156
+ ], {:legacy_support=>false, :updated_after=>nil}],
123
157
  })
124
158
  end
125
159
  end
@@ -128,13 +162,18 @@ RSpec.describe CanvasSync do
128
162
  it "appends the SyncAssignmentsJob" do
129
163
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
130
164
 
131
- expect(chain.chain_data).to eq(
132
- jobs: [
133
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
134
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
135
- { job: CanvasSync::Jobs::SyncAssignmentsJob.to_s, options: {} },
136
- ],
137
- global_options: { legacy_support: false },
165
+ expect(chain.normalize!).to eq(
166
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
167
+ :parameters => [[
168
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
169
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
170
+ :sub_jobs=>[
171
+ {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
172
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
173
+ ]
174
+ }]}
175
+ ]]}
176
+ ], {:legacy_support=>false, :updated_after=>nil}],
138
177
  )
139
178
  end
140
179
  end
@@ -143,13 +182,18 @@ RSpec.describe CanvasSync do
143
182
  it "appends the SyncSubmissionsJob" do
144
183
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
145
184
 
146
- expect(chain.chain_data).to eq(
147
- jobs: [
148
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
149
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
150
- { job: CanvasSync::Jobs::SyncSubmissionsJob.to_s, options: {} },
151
- ],
152
- global_options: { legacy_support: false },
185
+ expect(chain.normalize!).to eq(
186
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
187
+ :parameters => [[
188
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
189
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
190
+ :sub_jobs=>[
191
+ {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
192
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
193
+ ]
194
+ }]}
195
+ ]]}
196
+ ], {:legacy_support=>false, :updated_after=>nil}],
153
197
  )
154
198
  end
155
199
  end
@@ -158,95 +202,24 @@ RSpec.describe CanvasSync do
158
202
  it "appends the SyncAssignmentGroupsJob" do
159
203
  chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
160
204
 
161
- expect(chain.chain_data).to eq(
162
- jobs: [
163
- { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
164
- { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
165
- { job: CanvasSync::Jobs::SyncAssignmentGroupsJob.to_s, options: {} },
166
- ],
167
- global_options: { legacy_support: false },
205
+ expect(chain.normalize!).to eq(
206
+ :job => "CanvasSync::Jobs::BeginSyncChainJob",
207
+ :parameters => [[
208
+ {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
209
+ {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
210
+ :sub_jobs=>[
211
+ {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
212
+ {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
213
+ ]
214
+ }]}
215
+ ]]}
216
+ ], {:legacy_support=>false, :updated_after=>nil}],
168
217
  )
169
218
  end
170
219
  end
171
220
  end
172
221
  end
173
222
 
174
- describe ".simple_report_sync" do
175
-
176
- it 'invokes the first job in the queue and passes on the rest of the job chain' do
177
- expected_job_chain = CanvasSync.simple_report_chain(
178
- [
179
- {
180
- report_name: 'proservices_provisioning_csv',
181
- model: 'users',
182
- params: {
183
- "parameters[include_deleted]" => true,
184
- "parameters[users]" => true
185
- }
186
- }
187
- ]
188
- ).chain_data
189
- first_job = expected_job_chain[:jobs].shift
190
-
191
- expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
192
- .with(
193
- expected_job_chain,
194
- first_job[:options]
195
- )
196
-
197
- CanvasSync.simple_report_sync(
198
- [
199
- {
200
- report_name: 'proservices_provisioning_csv',
201
- model: 'users',
202
- params: {
203
- "parameters[include_deleted]" => true,
204
- "parameters[users]" => true
205
- }
206
- }
207
- ]
208
- )
209
- end
210
-
211
- it 'receives the job chain for the specified table' do
212
- chain = CanvasSync.simple_report_chain(
213
- [
214
- {
215
- report_name: 'proservices_provisioning_csv',
216
- model: 'users',
217
- params: {
218
- "parameters[include_deleted]" => true,
219
- "parameters[users]" => true
220
- }
221
- }
222
- ]
223
- )
224
-
225
- expected_job_chain = {
226
- jobs: [
227
- {
228
- job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
229
- options: {
230
- report_name: 'proservices_provisioning_csv',
231
- model: 'users',
232
- mapping: 'users',
233
- klass: 'User',
234
- term_scope: nil,
235
- params: {
236
- "parameters[include_deleted]" => true,
237
- "parameters[users]" => true
238
- }
239
- }
240
- }
241
- ],
242
- global_options: {}
243
- }
244
-
245
- expect(chain.chain_data).to eq(expected_job_chain)
246
-
247
- end
248
- end
249
-
250
223
  describe ".sync_scope" do
251
224
 
252
225
  end
@@ -1,14 +1,14 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  class GoodJob < CanvasSync::Job
4
- def perform(job_chain, argument)
4
+ def perform(argument)
5
5
  end
6
6
  end
7
7
 
8
8
  class EvilError < StandardError; end;
9
9
 
10
10
  class BadJob < CanvasSync::Job
11
- def perform(job_chain, argument)
11
+ def perform(argument)
12
12
  raise EvilError
13
13
  end
14
14
 
@@ -25,19 +25,19 @@ RSpec.describe CanvasSync::Job do
25
25
  describe '#perform' do
26
26
  it 'creates a CanvasSync::JobLog and logs relevant data on it' do
27
27
  expect {
28
- GoodJob.perform_now({}, "argument")
28
+ GoodJob.perform_now("argument")
29
29
  }.to change { CanvasSync::JobLog.count }.by(1)
30
30
 
31
31
  job_log = CanvasSync::JobLog.last
32
32
  expect(job_log.started_at).to_not be_nil
33
33
  expect(job_log.job_class).to eq(GoodJob.to_s)
34
- expect(job_log.job_arguments).to eq([{}, "argument"])
34
+ expect(job_log.job_arguments).to eq(["argument"])
35
35
  expect(job_log.completed_at).to_not be_nil
36
36
  end
37
37
 
38
38
  it 'logs exceptions on the CanvasSync::JobLog and then re-raises' do
39
39
  expect {
40
- BadJob.perform_now({}, "argument")
40
+ BadJob.perform_now("argument")
41
41
  }.to raise_exception(StandardError)
42
42
 
43
43
  job_log = CanvasSync::JobLog.last
@@ -47,25 +47,17 @@ RSpec.describe CanvasSync::Job do
47
47
 
48
48
  it 'invokes an error handler' do
49
49
  expect(BadJob).to receive(:good_catcher).once
50
+ set_batch_context(on_failure: 'BadJob.good_catcher')
50
51
  expect {
51
- BadJob.perform_now({
52
- jobs: [],
53
- global_options: {
54
- on_failure: 'BadJob.good_catcher'
55
- }
56
- })
52
+ BadJob.perform_now()
57
53
  }.to raise_exception(StandardError)
58
54
  end
59
55
 
60
56
  it 'logs a failing error handler' do
61
57
  expect(BadJob).to receive(:bad_catcher).once.and_call_original
58
+ set_batch_context(on_failure: 'BadJob.bad_catcher')
62
59
  expect {
63
- BadJob.perform_now({
64
- jobs: [],
65
- global_options: {
66
- on_failure: 'BadJob.bad_catcher'
67
- }
68
- })
60
+ BadJob.perform_now()
69
61
  }.to raise_exception(StandardError)
70
62
  job_log = CanvasSync::JobLog.last
71
63
  expect(job_log.backtrace).to include "Error Occurred while handling an Error"