canvas_sync 0.16.5 → 0.17.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +49 -137
- data/app/models/canvas_sync/sync_batch.rb +5 -0
- data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
- data/lib/canvas_sync/importers/bulk_importer.rb +4 -7
- data/lib/canvas_sync/job.rb +4 -10
- data/lib/canvas_sync/job_batches/batch.rb +399 -0
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
- data/lib/canvas_sync/job_batches/callback.rb +153 -0
- data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
- data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
- data/lib/canvas_sync/job_batches/status.rb +63 -0
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
- data/lib/canvas_sync/jobs/report_checker.rb +3 -6
- data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
- data/lib/canvas_sync/jobs/report_starter.rb +28 -20
- data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
- data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
- data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
- data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
- data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
- data/lib/canvas_sync/processors/assignment_groups_processor.rb +2 -3
- data/lib/canvas_sync/processors/assignments_processor.rb +2 -3
- data/lib/canvas_sync/processors/context_module_items_processor.rb +2 -3
- data/lib/canvas_sync/processors/context_modules_processor.rb +2 -3
- data/lib/canvas_sync/processors/normal_processor.rb +1 -2
- data/lib/canvas_sync/processors/provisioning_report_processor.rb +2 -10
- data/lib/canvas_sync/processors/submissions_processor.rb +2 -3
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +34 -97
- data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
- data/spec/canvas_sync/jobs/job_spec.rb +9 -17
- data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
- data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
- data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
- data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
- data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
- data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
- data/spec/dummy/config/environments/test.rb +2 -0
- data/spec/dummy/db/schema.rb +9 -1
- data/spec/job_batching/batch_aware_job_spec.rb +100 -0
- data/spec/job_batching/batch_spec.rb +363 -0
- data/spec/job_batching/callback_spec.rb +38 -0
- data/spec/job_batching/flow_spec.rb +91 -0
- data/spec/job_batching/integration/integration.rb +57 -0
- data/spec/job_batching/integration/nested.rb +88 -0
- data/spec/job_batching/integration/simple.rb +47 -0
- data/spec/job_batching/integration/workflow.rb +134 -0
- data/spec/job_batching/integration_helper.rb +48 -0
- data/spec/job_batching/sidekiq_spec.rb +124 -0
- data/spec/job_batching/status_spec.rb +92 -0
- data/spec/job_batching/support/base_job.rb +14 -0
- data/spec/job_batching/support/sample_callback.rb +2 -0
- data/spec/spec_helper.rb +10 -0
- metadata +91 -23
- data/lib/canvas_sync/job_chain.rb +0 -102
- data/lib/canvas_sync/jobs/fork_gather.rb +0 -74
- data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
- data/spec/dummy/db/test.sqlite3 +0 -0
- data/spec/dummy/log/development.log +0 -1248
- data/spec/dummy/log/test.log +0 -43258
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +0 -3
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +0 -4
data/lib/canvas_sync.rb
CHANGED
@@ -5,7 +5,6 @@ require "canvas_sync/engine"
|
|
5
5
|
require "canvas_sync/misc_helper"
|
6
6
|
require "canvas_sync/class_callback_executor"
|
7
7
|
require "canvas_sync/job"
|
8
|
-
require "canvas_sync/job_chain"
|
9
8
|
require "canvas_sync/sidekiq_job"
|
10
9
|
require "canvas_sync/api_syncable"
|
11
10
|
require "canvas_sync/record"
|
@@ -14,6 +13,8 @@ require "canvas_sync/jobs/report_checker"
|
|
14
13
|
require "canvas_sync/jobs/report_processor_job"
|
15
14
|
require "canvas_sync/config"
|
16
15
|
|
16
|
+
require "canvas_sync/job_batches/batch"
|
17
|
+
|
17
18
|
Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
|
18
19
|
Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
|
19
20
|
Dir[File.dirname(__FILE__) + "/canvas_sync/importers/*.rb"].each { |file| require file }
|
@@ -58,6 +59,9 @@ module CanvasSync
|
|
58
59
|
graded_submissions
|
59
60
|
].freeze
|
60
61
|
|
62
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
|
63
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
|
64
|
+
|
61
65
|
class << self
|
62
66
|
# Runs a standard provisioning sync job with no extra report types.
|
63
67
|
# Terms will be synced first using the API. If you are syncing users/roles/admins
|
@@ -73,51 +77,9 @@ module CanvasSync
|
|
73
77
|
# and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
|
74
78
|
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
75
79
|
# canvas_sync_client methods require an account ID.
|
76
|
-
def provisioning_sync(models,
|
80
|
+
def provisioning_sync(models, **kwargs)
|
77
81
|
validate_models!(models)
|
78
|
-
|
79
|
-
end
|
80
|
-
|
81
|
-
# Runs a report different from provisioning sync job with no extra report types.
|
82
|
-
#
|
83
|
-
# @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
|
84
|
-
# want to import:
|
85
|
-
# [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
|
86
|
-
# @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
|
87
|
-
# The provisioning report will be run for each of the terms contained in that scope.
|
88
|
-
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
89
|
-
# canvas_sync_client methods require an account ID.
|
90
|
-
def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
|
91
|
-
invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
|
92
|
-
end
|
93
|
-
|
94
|
-
# Runs a chain of ordered jobs
|
95
|
-
#
|
96
|
-
# See the README for usage and examples
|
97
|
-
#
|
98
|
-
# @param job_chain [Hash]
|
99
|
-
def process_jobs(job_chain)
|
100
|
-
invoke_next(job_chain)
|
101
|
-
end
|
102
|
-
|
103
|
-
# @deprecated
|
104
|
-
def duplicate_chain(job_chain)
|
105
|
-
Marshal.load(Marshal.dump(job_chain))
|
106
|
-
end
|
107
|
-
|
108
|
-
# Invokes the next job in a chain of jobs.
|
109
|
-
#
|
110
|
-
# This should typically be called automatically by the gem where necessary.
|
111
|
-
#
|
112
|
-
# @param job_chain [Hash] A chain of jobs to execute
|
113
|
-
def invoke_next(job_chain, extra_options: {})
|
114
|
-
job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
|
115
|
-
job_chain.perform_next(extra_options)
|
116
|
-
end
|
117
|
-
|
118
|
-
def fork(job_log, job_chain, keys: [], &blk)
|
119
|
-
job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
|
120
|
-
job_chain.fork(job_log, keys: keys, &blk)
|
82
|
+
default_provisioning_report_chain(models, **kwargs).process!
|
121
83
|
end
|
122
84
|
|
123
85
|
# Given a Model or Relation, scope it down to items that should be synced
|
@@ -134,33 +96,6 @@ module CanvasSync
|
|
134
96
|
scope
|
135
97
|
end
|
136
98
|
|
137
|
-
# Syn any report to an specific set of models
|
138
|
-
#
|
139
|
-
# @param reports_mapping [Array<Hash>] List of reports with their specific model and params
|
140
|
-
# @param term_scope [String]
|
141
|
-
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
142
|
-
# canvas_sync_client methods require an account ID.
|
143
|
-
def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
|
144
|
-
jobs = reports_mapping.map do |report|
|
145
|
-
{
|
146
|
-
job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
|
147
|
-
options: {
|
148
|
-
report_name: report[:report_name],
|
149
|
-
model: report[:model],
|
150
|
-
mapping: report[:model],
|
151
|
-
klass: report[:model].singularize.capitalize.to_s,
|
152
|
-
term_scope: term_scope,
|
153
|
-
params: report[:params]
|
154
|
-
}
|
155
|
-
}
|
156
|
-
end
|
157
|
-
|
158
|
-
global_options = {}
|
159
|
-
global_options[:account_id] = account_id if account_id.present?
|
160
|
-
|
161
|
-
JobChain.new(jobs: jobs, global_options: global_options)
|
162
|
-
end
|
163
|
-
|
164
99
|
# Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
|
165
100
|
#
|
166
101
|
# @param models [Array<String>]
|
@@ -171,7 +106,7 @@ module CanvasSync
|
|
171
106
|
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
172
107
|
# canvas_sync_client methods require an account ID.
|
173
108
|
# @return [Hash]
|
174
|
-
def default_provisioning_report_chain(models, term_scope
|
109
|
+
def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
|
175
110
|
return unless models.present?
|
176
111
|
models.map! &:to_s
|
177
112
|
term_scope = term_scope.to_s if term_scope
|
@@ -190,64 +125,66 @@ module CanvasSync
|
|
190
125
|
context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
|
191
126
|
}.with_indifferent_access
|
192
127
|
|
193
|
-
|
128
|
+
root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
|
129
|
+
concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
|
130
|
+
root_chain << concurrent_root_chain
|
131
|
+
current_chain = concurrent_root_chain
|
132
|
+
|
194
133
|
try_add_model_job = ->(model) {
|
195
134
|
return unless models.include?(model)
|
196
|
-
|
135
|
+
current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
|
197
136
|
models -= [model]
|
198
137
|
}
|
199
138
|
|
200
139
|
##############################
|
201
|
-
#
|
140
|
+
# General provisioning jobs (not term-scoped)
|
202
141
|
##############################
|
203
142
|
|
204
|
-
#
|
205
|
-
models
|
206
|
-
try_add_model_job.call('terms')
|
143
|
+
# Terms are always synced regardless of the models option
|
144
|
+
models -= ['terms']
|
207
145
|
|
208
|
-
# Accounts, users, roles, and admins
|
146
|
+
# Accounts, users, roles, and admins cannot be scoped to term
|
209
147
|
try_add_model_job.call('accounts')
|
210
148
|
|
211
149
|
# These Models use the provisioning report, but are not term-scoped,
|
212
|
-
# so we sync them
|
150
|
+
# so we sync them outside of the term scoping to ensure work is not duplicated
|
213
151
|
if term_scope.present?
|
214
152
|
models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
|
215
|
-
|
216
|
-
generate_provisioning_jobs(first_provisioning_models, options)
|
217
|
-
)
|
153
|
+
current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
|
218
154
|
end
|
219
155
|
|
220
156
|
try_add_model_job.call('roles')
|
221
157
|
try_add_model_job.call('admins')
|
222
|
-
pre_provisioning_jobs = jobs
|
223
158
|
|
224
159
|
###############################
|
225
|
-
#
|
160
|
+
# Per-term provisioning jobs
|
226
161
|
###############################
|
227
162
|
|
228
|
-
|
163
|
+
per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
|
164
|
+
current_chain << per_term_chain
|
165
|
+
current_chain = per_term_chain
|
166
|
+
|
229
167
|
try_add_model_job.call('assignments')
|
230
168
|
try_add_model_job.call('submissions')
|
231
169
|
try_add_model_job.call('assignment_groups')
|
232
170
|
try_add_model_job.call('context_modules')
|
233
171
|
try_add_model_job.call('context_module_items')
|
234
|
-
|
172
|
+
|
173
|
+
current_chain.insert(
|
174
|
+
generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users'])
|
175
|
+
)
|
235
176
|
|
236
177
|
###############################
|
237
|
-
#
|
178
|
+
# Wrap it all up
|
238
179
|
###############################
|
239
180
|
|
240
|
-
|
241
|
-
*pre_provisioning_jobs,
|
242
|
-
*generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
|
243
|
-
*post_provisioning_jobs,
|
244
|
-
]
|
245
|
-
|
246
|
-
global_options = { legacy_support: legacy_support }
|
181
|
+
global_options = { legacy_support: legacy_support, updated_after: updated_after }
|
247
182
|
global_options[:account_id] = account_id if account_id.present?
|
248
183
|
global_options.merge!(options[:global]) if options[:global].present?
|
249
184
|
|
250
|
-
|
185
|
+
root_chain.params[1] = global_options
|
186
|
+
|
187
|
+
root_chain
|
251
188
|
end
|
252
189
|
|
253
190
|
def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
|
@@ -3,13 +3,12 @@ require 'spec_helper'
|
|
3
3
|
RSpec.describe CanvasSync do
|
4
4
|
describe '.provisioning_sync' do
|
5
5
|
it 'invokes the first job in the queue and passes on the rest of the job chain' do
|
6
|
-
expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil).
|
7
|
-
first_job = expected_job_chain[:jobs].shift
|
6
|
+
expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:parameters][0]
|
8
7
|
|
9
|
-
expect(CanvasSync::Jobs::
|
8
|
+
expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
|
10
9
|
.with(
|
11
10
|
expected_job_chain,
|
12
|
-
|
11
|
+
anything
|
13
12
|
)
|
14
13
|
|
15
14
|
CanvasSync.provisioning_sync(['courses'])
|
@@ -25,32 +24,42 @@ RSpec.describe CanvasSync do
|
|
25
24
|
|
26
25
|
describe '.default_provisioning_report_chain' do
|
27
26
|
it 'splits an options: Hash into options for separate models' do
|
28
|
-
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active, options: {
|
27
|
+
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active, options: {
|
29
28
|
terms: { a: 1 },
|
30
29
|
users: { b: 2 },
|
31
30
|
provisioning: { c: 3 },
|
32
31
|
global: { d: 4 },
|
33
32
|
})
|
34
|
-
expect(chain.
|
35
|
-
|
36
|
-
|
37
|
-
{
|
38
|
-
|
39
|
-
|
40
|
-
|
33
|
+
expect(chain.normalize!).to eq({
|
34
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
35
|
+
:parameters => [[
|
36
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
37
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
|
38
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
39
|
+
:sub_jobs=>[
|
40
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active", :c=>3}}
|
41
|
+
]
|
42
|
+
}]}
|
43
|
+
]]}
|
44
|
+
], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
|
41
45
|
})
|
42
46
|
end
|
43
47
|
|
44
48
|
context 'we are syncing users with a term scope' do
|
45
49
|
it 'syncs the users in a separate job that runs first' do
|
46
|
-
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active)
|
47
|
-
expect(chain.
|
48
|
-
|
49
|
-
|
50
|
-
{
|
51
|
-
|
52
|
-
|
53
|
-
|
50
|
+
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
|
51
|
+
expect(chain.normalize!).to eq({
|
52
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
53
|
+
:parameters => [[
|
54
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
55
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
|
56
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
57
|
+
:sub_jobs=>[
|
58
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
59
|
+
]
|
60
|
+
}]}
|
61
|
+
]]}
|
62
|
+
], {:legacy_support=>false, :updated_after=>nil}]
|
54
63
|
})
|
55
64
|
end
|
56
65
|
end
|
@@ -58,26 +67,36 @@ RSpec.describe CanvasSync do
|
|
58
67
|
context 'we are syncing users without a term scope' do
|
59
68
|
it 'syncs users along with the rest of the provisioning report' do
|
60
69
|
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
|
61
|
-
expect(chain.
|
62
|
-
|
63
|
-
|
64
|
-
{
|
65
|
-
|
66
|
-
|
70
|
+
expect(chain.normalize!).to eq({
|
71
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
72
|
+
:parameters => [[
|
73
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
74
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
75
|
+
:sub_jobs=>[
|
76
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"], :term_scope=>nil}}
|
77
|
+
]
|
78
|
+
}]}
|
79
|
+
]]}
|
80
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
67
81
|
})
|
68
82
|
end
|
69
83
|
end
|
70
84
|
|
71
85
|
context 'we are syncing roles with a term scope' do
|
72
86
|
it 'syncs the roles in a separate job that runs first' do
|
73
|
-
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], :active)
|
74
|
-
expect(chain.
|
75
|
-
|
76
|
-
|
77
|
-
{
|
78
|
-
|
79
|
-
|
80
|
-
|
87
|
+
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
|
88
|
+
expect(chain.normalize!).to eq({
|
89
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
90
|
+
:parameters => [[
|
91
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
92
|
+
{:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
|
93
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
94
|
+
:sub_jobs=>[
|
95
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
96
|
+
]
|
97
|
+
}]}
|
98
|
+
]]}
|
99
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
81
100
|
})
|
82
101
|
end
|
83
102
|
end
|
@@ -85,27 +104,37 @@ RSpec.describe CanvasSync do
|
|
85
104
|
context 'we are syncing roles without a term scope' do
|
86
105
|
it 'syncs roles separately even with no term scope' do
|
87
106
|
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
|
88
|
-
expect(chain.
|
89
|
-
|
90
|
-
|
91
|
-
{
|
92
|
-
|
93
|
-
|
94
|
-
|
107
|
+
expect(chain.normalize!).to eq({
|
108
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
109
|
+
:parameters => [[
|
110
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
111
|
+
{:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
|
112
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
113
|
+
:sub_jobs=>[
|
114
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
|
115
|
+
]
|
116
|
+
}]}
|
117
|
+
]]}
|
118
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
95
119
|
})
|
96
120
|
end
|
97
121
|
end
|
98
122
|
|
99
123
|
context 'we are syncing admins with a term scope' do
|
100
124
|
it 'syncs the admins in a separate job that runs first' do
|
101
|
-
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], :active)
|
102
|
-
expect(chain.
|
103
|
-
|
104
|
-
|
105
|
-
{
|
106
|
-
|
107
|
-
|
108
|
-
|
125
|
+
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
|
126
|
+
expect(chain.normalize!).to eq({
|
127
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
128
|
+
:parameters => [[
|
129
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
130
|
+
{:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
|
131
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
132
|
+
:sub_jobs=>[
|
133
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
134
|
+
]
|
135
|
+
}]}
|
136
|
+
]]}
|
137
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
109
138
|
})
|
110
139
|
end
|
111
140
|
end
|
@@ -113,13 +142,18 @@ RSpec.describe CanvasSync do
|
|
113
142
|
context 'we are syncing admins without a term scope' do
|
114
143
|
it 'syncs admins separately even with no term scope' do
|
115
144
|
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
|
116
|
-
expect(chain.
|
117
|
-
|
118
|
-
|
119
|
-
{
|
120
|
-
|
121
|
-
|
122
|
-
|
145
|
+
expect(chain.normalize!).to eq({
|
146
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
147
|
+
:parameters => [[
|
148
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
149
|
+
{:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
|
150
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
151
|
+
:sub_jobs=>[
|
152
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
|
153
|
+
]
|
154
|
+
}]}
|
155
|
+
]]}
|
156
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
123
157
|
})
|
124
158
|
end
|
125
159
|
end
|
@@ -128,13 +162,18 @@ RSpec.describe CanvasSync do
|
|
128
162
|
it "appends the SyncAssignmentsJob" do
|
129
163
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
|
130
164
|
|
131
|
-
expect(chain.
|
132
|
-
|
133
|
-
|
134
|
-
{
|
135
|
-
|
136
|
-
|
137
|
-
|
165
|
+
expect(chain.normalize!).to eq(
|
166
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
167
|
+
:parameters => [[
|
168
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
169
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
170
|
+
:sub_jobs=>[
|
171
|
+
{:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
|
172
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
173
|
+
]
|
174
|
+
}]}
|
175
|
+
]]}
|
176
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
138
177
|
)
|
139
178
|
end
|
140
179
|
end
|
@@ -143,13 +182,18 @@ RSpec.describe CanvasSync do
|
|
143
182
|
it "appends the SyncSubmissionsJob" do
|
144
183
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
|
145
184
|
|
146
|
-
expect(chain.
|
147
|
-
|
148
|
-
|
149
|
-
{
|
150
|
-
|
151
|
-
|
152
|
-
|
185
|
+
expect(chain.normalize!).to eq(
|
186
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
187
|
+
:parameters => [[
|
188
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
189
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
190
|
+
:sub_jobs=>[
|
191
|
+
{:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
|
192
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
193
|
+
]
|
194
|
+
}]}
|
195
|
+
]]}
|
196
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
153
197
|
)
|
154
198
|
end
|
155
199
|
end
|
@@ -158,95 +202,24 @@ RSpec.describe CanvasSync do
|
|
158
202
|
it "appends the SyncAssignmentGroupsJob" do
|
159
203
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
|
160
204
|
|
161
|
-
expect(chain.
|
162
|
-
|
163
|
-
|
164
|
-
{
|
165
|
-
|
166
|
-
|
167
|
-
|
205
|
+
expect(chain.normalize!).to eq(
|
206
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
207
|
+
:parameters => [[
|
208
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
209
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
210
|
+
:sub_jobs=>[
|
211
|
+
{:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
|
212
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
213
|
+
]
|
214
|
+
}]}
|
215
|
+
]]}
|
216
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
168
217
|
)
|
169
218
|
end
|
170
219
|
end
|
171
220
|
end
|
172
221
|
end
|
173
222
|
|
174
|
-
describe ".simple_report_sync" do
|
175
|
-
|
176
|
-
it 'invokes the first job in the queue and passes on the rest of the job chain' do
|
177
|
-
expected_job_chain = CanvasSync.simple_report_chain(
|
178
|
-
[
|
179
|
-
{
|
180
|
-
report_name: 'proservices_provisioning_csv',
|
181
|
-
model: 'users',
|
182
|
-
params: {
|
183
|
-
"parameters[include_deleted]" => true,
|
184
|
-
"parameters[users]" => true
|
185
|
-
}
|
186
|
-
}
|
187
|
-
]
|
188
|
-
).chain_data
|
189
|
-
first_job = expected_job_chain[:jobs].shift
|
190
|
-
|
191
|
-
expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
|
192
|
-
.with(
|
193
|
-
expected_job_chain,
|
194
|
-
first_job[:options]
|
195
|
-
)
|
196
|
-
|
197
|
-
CanvasSync.simple_report_sync(
|
198
|
-
[
|
199
|
-
{
|
200
|
-
report_name: 'proservices_provisioning_csv',
|
201
|
-
model: 'users',
|
202
|
-
params: {
|
203
|
-
"parameters[include_deleted]" => true,
|
204
|
-
"parameters[users]" => true
|
205
|
-
}
|
206
|
-
}
|
207
|
-
]
|
208
|
-
)
|
209
|
-
end
|
210
|
-
|
211
|
-
it 'receives the job chain for the specified table' do
|
212
|
-
chain = CanvasSync.simple_report_chain(
|
213
|
-
[
|
214
|
-
{
|
215
|
-
report_name: 'proservices_provisioning_csv',
|
216
|
-
model: 'users',
|
217
|
-
params: {
|
218
|
-
"parameters[include_deleted]" => true,
|
219
|
-
"parameters[users]" => true
|
220
|
-
}
|
221
|
-
}
|
222
|
-
]
|
223
|
-
)
|
224
|
-
|
225
|
-
expected_job_chain = {
|
226
|
-
jobs: [
|
227
|
-
{
|
228
|
-
job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
|
229
|
-
options: {
|
230
|
-
report_name: 'proservices_provisioning_csv',
|
231
|
-
model: 'users',
|
232
|
-
mapping: 'users',
|
233
|
-
klass: 'User',
|
234
|
-
term_scope: nil,
|
235
|
-
params: {
|
236
|
-
"parameters[include_deleted]" => true,
|
237
|
-
"parameters[users]" => true
|
238
|
-
}
|
239
|
-
}
|
240
|
-
}
|
241
|
-
],
|
242
|
-
global_options: {}
|
243
|
-
}
|
244
|
-
|
245
|
-
expect(chain.chain_data).to eq(expected_job_chain)
|
246
|
-
|
247
|
-
end
|
248
|
-
end
|
249
|
-
|
250
223
|
describe ".sync_scope" do
|
251
224
|
|
252
225
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
class GoodJob < CanvasSync::Job
|
4
|
-
def perform(
|
4
|
+
def perform(argument)
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
8
|
class EvilError < StandardError; end;
|
9
9
|
|
10
10
|
class BadJob < CanvasSync::Job
|
11
|
-
def perform(
|
11
|
+
def perform(argument)
|
12
12
|
raise EvilError
|
13
13
|
end
|
14
14
|
|
@@ -25,19 +25,19 @@ RSpec.describe CanvasSync::Job do
|
|
25
25
|
describe '#perform' do
|
26
26
|
it 'creates a CanvasSync::JobLog and logs relevant data on it' do
|
27
27
|
expect {
|
28
|
-
GoodJob.perform_now(
|
28
|
+
GoodJob.perform_now("argument")
|
29
29
|
}.to change { CanvasSync::JobLog.count }.by(1)
|
30
30
|
|
31
31
|
job_log = CanvasSync::JobLog.last
|
32
32
|
expect(job_log.started_at).to_not be_nil
|
33
33
|
expect(job_log.job_class).to eq(GoodJob.to_s)
|
34
|
-
expect(job_log.job_arguments).to eq([
|
34
|
+
expect(job_log.job_arguments).to eq(["argument"])
|
35
35
|
expect(job_log.completed_at).to_not be_nil
|
36
36
|
end
|
37
37
|
|
38
38
|
it 'logs exceptions on the CanvasSync::JobLog and then re-raises' do
|
39
39
|
expect {
|
40
|
-
BadJob.perform_now(
|
40
|
+
BadJob.perform_now("argument")
|
41
41
|
}.to raise_exception(StandardError)
|
42
42
|
|
43
43
|
job_log = CanvasSync::JobLog.last
|
@@ -47,25 +47,17 @@ RSpec.describe CanvasSync::Job do
|
|
47
47
|
|
48
48
|
it 'invokes an error handler' do
|
49
49
|
expect(BadJob).to receive(:good_catcher).once
|
50
|
+
set_batch_context(on_failure: 'BadJob.good_catcher')
|
50
51
|
expect {
|
51
|
-
BadJob.perform_now(
|
52
|
-
jobs: [],
|
53
|
-
global_options: {
|
54
|
-
on_failure: 'BadJob.good_catcher'
|
55
|
-
}
|
56
|
-
})
|
52
|
+
BadJob.perform_now()
|
57
53
|
}.to raise_exception(StandardError)
|
58
54
|
end
|
59
55
|
|
60
56
|
it 'logs a failing error handler' do
|
61
57
|
expect(BadJob).to receive(:bad_catcher).once.and_call_original
|
58
|
+
set_batch_context(on_failure: 'BadJob.bad_catcher')
|
62
59
|
expect {
|
63
|
-
BadJob.perform_now(
|
64
|
-
jobs: [],
|
65
|
-
global_options: {
|
66
|
-
on_failure: 'BadJob.bad_catcher'
|
67
|
-
}
|
68
|
-
})
|
60
|
+
BadJob.perform_now()
|
69
61
|
}.to raise_exception(StandardError)
|
70
62
|
job_log = CanvasSync::JobLog.last
|
71
63
|
expect(job_log.backtrace).to include "Error Occurred while handling an Error"
|