canvas_sync 0.16.5 → 0.17.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +49 -137
- data/app/models/canvas_sync/sync_batch.rb +5 -0
- data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
- data/lib/canvas_sync/importers/bulk_importer.rb +4 -7
- data/lib/canvas_sync/job.rb +4 -10
- data/lib/canvas_sync/job_batches/batch.rb +399 -0
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
- data/lib/canvas_sync/job_batches/callback.rb +153 -0
- data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
- data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
- data/lib/canvas_sync/job_batches/status.rb +63 -0
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
- data/lib/canvas_sync/jobs/report_checker.rb +3 -6
- data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
- data/lib/canvas_sync/jobs/report_starter.rb +28 -20
- data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
- data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
- data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
- data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
- data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
- data/lib/canvas_sync/processors/assignment_groups_processor.rb +2 -3
- data/lib/canvas_sync/processors/assignments_processor.rb +2 -3
- data/lib/canvas_sync/processors/context_module_items_processor.rb +2 -3
- data/lib/canvas_sync/processors/context_modules_processor.rb +2 -3
- data/lib/canvas_sync/processors/normal_processor.rb +1 -2
- data/lib/canvas_sync/processors/provisioning_report_processor.rb +2 -10
- data/lib/canvas_sync/processors/submissions_processor.rb +2 -3
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +34 -97
- data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
- data/spec/canvas_sync/jobs/job_spec.rb +9 -17
- data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
- data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
- data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
- data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
- data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
- data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
- data/spec/dummy/config/environments/test.rb +2 -0
- data/spec/dummy/db/schema.rb +9 -1
- data/spec/job_batching/batch_aware_job_spec.rb +100 -0
- data/spec/job_batching/batch_spec.rb +363 -0
- data/spec/job_batching/callback_spec.rb +38 -0
- data/spec/job_batching/flow_spec.rb +91 -0
- data/spec/job_batching/integration/integration.rb +57 -0
- data/spec/job_batching/integration/nested.rb +88 -0
- data/spec/job_batching/integration/simple.rb +47 -0
- data/spec/job_batching/integration/workflow.rb +134 -0
- data/spec/job_batching/integration_helper.rb +48 -0
- data/spec/job_batching/sidekiq_spec.rb +124 -0
- data/spec/job_batching/status_spec.rb +92 -0
- data/spec/job_batching/support/base_job.rb +14 -0
- data/spec/job_batching/support/sample_callback.rb +2 -0
- data/spec/spec_helper.rb +10 -0
- metadata +91 -23
- data/lib/canvas_sync/job_chain.rb +0 -102
- data/lib/canvas_sync/jobs/fork_gather.rb +0 -74
- data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
- data/spec/dummy/db/test.sqlite3 +0 -0
- data/spec/dummy/log/development.log +0 -1248
- data/spec/dummy/log/test.log +0 -43258
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +0 -3
- data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +0 -4
data/lib/canvas_sync.rb
CHANGED
@@ -5,7 +5,6 @@ require "canvas_sync/engine"
|
|
5
5
|
require "canvas_sync/misc_helper"
|
6
6
|
require "canvas_sync/class_callback_executor"
|
7
7
|
require "canvas_sync/job"
|
8
|
-
require "canvas_sync/job_chain"
|
9
8
|
require "canvas_sync/sidekiq_job"
|
10
9
|
require "canvas_sync/api_syncable"
|
11
10
|
require "canvas_sync/record"
|
@@ -14,6 +13,8 @@ require "canvas_sync/jobs/report_checker"
|
|
14
13
|
require "canvas_sync/jobs/report_processor_job"
|
15
14
|
require "canvas_sync/config"
|
16
15
|
|
16
|
+
require "canvas_sync/job_batches/batch"
|
17
|
+
|
17
18
|
Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
|
18
19
|
Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
|
19
20
|
Dir[File.dirname(__FILE__) + "/canvas_sync/importers/*.rb"].each { |file| require file }
|
@@ -58,6 +59,9 @@ module CanvasSync
|
|
58
59
|
graded_submissions
|
59
60
|
].freeze
|
60
61
|
|
62
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::SyncTermsJob, :sub_jobs)
|
63
|
+
JobBatches::ChainBuilder.register_chain_job(CanvasSync::Jobs::BeginSyncChainJob, 0)
|
64
|
+
|
61
65
|
class << self
|
62
66
|
# Runs a standard provisioning sync job with no extra report types.
|
63
67
|
# Terms will be synced first using the API. If you are syncing users/roles/admins
|
@@ -73,51 +77,9 @@ module CanvasSync
|
|
73
77
|
# and inserts it into the database. If an array of model names is provided then only those models will use legacy support.
|
74
78
|
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
75
79
|
# canvas_sync_client methods require an account ID.
|
76
|
-
def provisioning_sync(models,
|
80
|
+
def provisioning_sync(models, **kwargs)
|
77
81
|
validate_models!(models)
|
78
|
-
|
79
|
-
end
|
80
|
-
|
81
|
-
# Runs a report different from provisioning sync job with no extra report types.
|
82
|
-
#
|
83
|
-
# @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
|
84
|
-
# want to import:
|
85
|
-
# [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
|
86
|
-
# @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
|
87
|
-
# The provisioning report will be run for each of the terms contained in that scope.
|
88
|
-
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
89
|
-
# canvas_sync_client methods require an account ID.
|
90
|
-
def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
|
91
|
-
invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
|
92
|
-
end
|
93
|
-
|
94
|
-
# Runs a chain of ordered jobs
|
95
|
-
#
|
96
|
-
# See the README for usage and examples
|
97
|
-
#
|
98
|
-
# @param job_chain [Hash]
|
99
|
-
def process_jobs(job_chain)
|
100
|
-
invoke_next(job_chain)
|
101
|
-
end
|
102
|
-
|
103
|
-
# @deprecated
|
104
|
-
def duplicate_chain(job_chain)
|
105
|
-
Marshal.load(Marshal.dump(job_chain))
|
106
|
-
end
|
107
|
-
|
108
|
-
# Invokes the next job in a chain of jobs.
|
109
|
-
#
|
110
|
-
# This should typically be called automatically by the gem where necessary.
|
111
|
-
#
|
112
|
-
# @param job_chain [Hash] A chain of jobs to execute
|
113
|
-
def invoke_next(job_chain, extra_options: {})
|
114
|
-
job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
|
115
|
-
job_chain.perform_next(extra_options)
|
116
|
-
end
|
117
|
-
|
118
|
-
def fork(job_log, job_chain, keys: [], &blk)
|
119
|
-
job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
|
120
|
-
job_chain.fork(job_log, keys: keys, &blk)
|
82
|
+
default_provisioning_report_chain(models, **kwargs).process!
|
121
83
|
end
|
122
84
|
|
123
85
|
# Given a Model or Relation, scope it down to items that should be synced
|
@@ -134,33 +96,6 @@ module CanvasSync
|
|
134
96
|
scope
|
135
97
|
end
|
136
98
|
|
137
|
-
# Syn any report to an specific set of models
|
138
|
-
#
|
139
|
-
# @param reports_mapping [Array<Hash>] List of reports with their specific model and params
|
140
|
-
# @param term_scope [String]
|
141
|
-
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
142
|
-
# canvas_sync_client methods require an account ID.
|
143
|
-
def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
|
144
|
-
jobs = reports_mapping.map do |report|
|
145
|
-
{
|
146
|
-
job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
|
147
|
-
options: {
|
148
|
-
report_name: report[:report_name],
|
149
|
-
model: report[:model],
|
150
|
-
mapping: report[:model],
|
151
|
-
klass: report[:model].singularize.capitalize.to_s,
|
152
|
-
term_scope: term_scope,
|
153
|
-
params: report[:params]
|
154
|
-
}
|
155
|
-
}
|
156
|
-
end
|
157
|
-
|
158
|
-
global_options = {}
|
159
|
-
global_options[:account_id] = account_id if account_id.present?
|
160
|
-
|
161
|
-
JobChain.new(jobs: jobs, global_options: global_options)
|
162
|
-
end
|
163
|
-
|
164
99
|
# Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
|
165
100
|
#
|
166
101
|
# @param models [Array<String>]
|
@@ -171,7 +106,7 @@ module CanvasSync
|
|
171
106
|
# @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
|
172
107
|
# canvas_sync_client methods require an account ID.
|
173
108
|
# @return [Hash]
|
174
|
-
def default_provisioning_report_chain(models, term_scope
|
109
|
+
def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
|
175
110
|
return unless models.present?
|
176
111
|
models.map! &:to_s
|
177
112
|
term_scope = term_scope.to_s if term_scope
|
@@ -190,64 +125,66 @@ module CanvasSync
|
|
190
125
|
context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
|
191
126
|
}.with_indifferent_access
|
192
127
|
|
193
|
-
|
128
|
+
root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
|
129
|
+
concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
|
130
|
+
root_chain << concurrent_root_chain
|
131
|
+
current_chain = concurrent_root_chain
|
132
|
+
|
194
133
|
try_add_model_job = ->(model) {
|
195
134
|
return unless models.include?(model)
|
196
|
-
|
135
|
+
current_chain << { job: model_job_map[model].to_s, options: options[model.to_sym] || {} }
|
197
136
|
models -= [model]
|
198
137
|
}
|
199
138
|
|
200
139
|
##############################
|
201
|
-
#
|
140
|
+
# General provisioning jobs (not term-scoped)
|
202
141
|
##############################
|
203
142
|
|
204
|
-
#
|
205
|
-
models
|
206
|
-
try_add_model_job.call('terms')
|
143
|
+
# Terms are always synced regardless of the models option
|
144
|
+
models -= ['terms']
|
207
145
|
|
208
|
-
# Accounts, users, roles, and admins
|
146
|
+
# Accounts, users, roles, and admins cannot be scoped to term
|
209
147
|
try_add_model_job.call('accounts')
|
210
148
|
|
211
149
|
# These Models use the provisioning report, but are not term-scoped,
|
212
|
-
# so we sync them
|
150
|
+
# so we sync them outside of the term scoping to ensure work is not duplicated
|
213
151
|
if term_scope.present?
|
214
152
|
models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
|
215
|
-
|
216
|
-
generate_provisioning_jobs(first_provisioning_models, options)
|
217
|
-
)
|
153
|
+
current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
|
218
154
|
end
|
219
155
|
|
220
156
|
try_add_model_job.call('roles')
|
221
157
|
try_add_model_job.call('admins')
|
222
|
-
pre_provisioning_jobs = jobs
|
223
158
|
|
224
159
|
###############################
|
225
|
-
#
|
160
|
+
# Per-term provisioning jobs
|
226
161
|
###############################
|
227
162
|
|
228
|
-
|
163
|
+
per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
|
164
|
+
current_chain << per_term_chain
|
165
|
+
current_chain = per_term_chain
|
166
|
+
|
229
167
|
try_add_model_job.call('assignments')
|
230
168
|
try_add_model_job.call('submissions')
|
231
169
|
try_add_model_job.call('assignment_groups')
|
232
170
|
try_add_model_job.call('context_modules')
|
233
171
|
try_add_model_job.call('context_module_items')
|
234
|
-
|
172
|
+
|
173
|
+
current_chain.insert(
|
174
|
+
generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users'])
|
175
|
+
)
|
235
176
|
|
236
177
|
###############################
|
237
|
-
#
|
178
|
+
# Wrap it all up
|
238
179
|
###############################
|
239
180
|
|
240
|
-
|
241
|
-
*pre_provisioning_jobs,
|
242
|
-
*generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
|
243
|
-
*post_provisioning_jobs,
|
244
|
-
]
|
245
|
-
|
246
|
-
global_options = { legacy_support: legacy_support }
|
181
|
+
global_options = { legacy_support: legacy_support, updated_after: updated_after }
|
247
182
|
global_options[:account_id] = account_id if account_id.present?
|
248
183
|
global_options.merge!(options[:global]) if options[:global].present?
|
249
184
|
|
250
|
-
|
185
|
+
root_chain.params[1] = global_options
|
186
|
+
|
187
|
+
root_chain
|
251
188
|
end
|
252
189
|
|
253
190
|
def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
|
@@ -3,13 +3,12 @@ require 'spec_helper'
|
|
3
3
|
RSpec.describe CanvasSync do
|
4
4
|
describe '.provisioning_sync' do
|
5
5
|
it 'invokes the first job in the queue and passes on the rest of the job chain' do
|
6
|
-
expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil).
|
7
|
-
first_job = expected_job_chain[:jobs].shift
|
6
|
+
expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:parameters][0]
|
8
7
|
|
9
|
-
expect(CanvasSync::Jobs::
|
8
|
+
expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
|
10
9
|
.with(
|
11
10
|
expected_job_chain,
|
12
|
-
|
11
|
+
anything
|
13
12
|
)
|
14
13
|
|
15
14
|
CanvasSync.provisioning_sync(['courses'])
|
@@ -25,32 +24,42 @@ RSpec.describe CanvasSync do
|
|
25
24
|
|
26
25
|
describe '.default_provisioning_report_chain' do
|
27
26
|
it 'splits an options: Hash into options for separate models' do
|
28
|
-
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active, options: {
|
27
|
+
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active, options: {
|
29
28
|
terms: { a: 1 },
|
30
29
|
users: { b: 2 },
|
31
30
|
provisioning: { c: 3 },
|
32
31
|
global: { d: 4 },
|
33
32
|
})
|
34
|
-
expect(chain.
|
35
|
-
|
36
|
-
|
37
|
-
{
|
38
|
-
|
39
|
-
|
40
|
-
|
33
|
+
expect(chain.normalize!).to eq({
|
34
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
35
|
+
:parameters => [[
|
36
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
37
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
|
38
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
39
|
+
:sub_jobs=>[
|
40
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active", :c=>3}}
|
41
|
+
]
|
42
|
+
}]}
|
43
|
+
]]}
|
44
|
+
], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
|
41
45
|
})
|
42
46
|
end
|
43
47
|
|
44
48
|
context 'we are syncing users with a term scope' do
|
45
49
|
it 'syncs the users in a separate job that runs first' do
|
46
|
-
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active)
|
47
|
-
expect(chain.
|
48
|
-
|
49
|
-
|
50
|
-
{
|
51
|
-
|
52
|
-
|
53
|
-
|
50
|
+
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
|
51
|
+
expect(chain.normalize!).to eq({
|
52
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
53
|
+
:parameters => [[
|
54
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
55
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
|
56
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
57
|
+
:sub_jobs=>[
|
58
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
59
|
+
]
|
60
|
+
}]}
|
61
|
+
]]}
|
62
|
+
], {:legacy_support=>false, :updated_after=>nil}]
|
54
63
|
})
|
55
64
|
end
|
56
65
|
end
|
@@ -58,26 +67,36 @@ RSpec.describe CanvasSync do
|
|
58
67
|
context 'we are syncing users without a term scope' do
|
59
68
|
it 'syncs users along with the rest of the provisioning report' do
|
60
69
|
chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
|
61
|
-
expect(chain.
|
62
|
-
|
63
|
-
|
64
|
-
{
|
65
|
-
|
66
|
-
|
70
|
+
expect(chain.normalize!).to eq({
|
71
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
72
|
+
:parameters => [[
|
73
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
74
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
75
|
+
:sub_jobs=>[
|
76
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"], :term_scope=>nil}}
|
77
|
+
]
|
78
|
+
}]}
|
79
|
+
]]}
|
80
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
67
81
|
})
|
68
82
|
end
|
69
83
|
end
|
70
84
|
|
71
85
|
context 'we are syncing roles with a term scope' do
|
72
86
|
it 'syncs the roles in a separate job that runs first' do
|
73
|
-
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], :active)
|
74
|
-
expect(chain.
|
75
|
-
|
76
|
-
|
77
|
-
{
|
78
|
-
|
79
|
-
|
80
|
-
|
87
|
+
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
|
88
|
+
expect(chain.normalize!).to eq({
|
89
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
90
|
+
:parameters => [[
|
91
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
92
|
+
{:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
|
93
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
94
|
+
:sub_jobs=>[
|
95
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
96
|
+
]
|
97
|
+
}]}
|
98
|
+
]]}
|
99
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
81
100
|
})
|
82
101
|
end
|
83
102
|
end
|
@@ -85,27 +104,37 @@ RSpec.describe CanvasSync do
|
|
85
104
|
context 'we are syncing roles without a term scope' do
|
86
105
|
it 'syncs roles separately even with no term scope' do
|
87
106
|
chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
|
88
|
-
expect(chain.
|
89
|
-
|
90
|
-
|
91
|
-
{
|
92
|
-
|
93
|
-
|
94
|
-
|
107
|
+
expect(chain.normalize!).to eq({
|
108
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
109
|
+
:parameters => [[
|
110
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
111
|
+
{:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
|
112
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
113
|
+
:sub_jobs=>[
|
114
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
|
115
|
+
]
|
116
|
+
}]}
|
117
|
+
]]}
|
118
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
95
119
|
})
|
96
120
|
end
|
97
121
|
end
|
98
122
|
|
99
123
|
context 'we are syncing admins with a term scope' do
|
100
124
|
it 'syncs the admins in a separate job that runs first' do
|
101
|
-
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], :active)
|
102
|
-
expect(chain.
|
103
|
-
|
104
|
-
|
105
|
-
{
|
106
|
-
|
107
|
-
|
108
|
-
|
125
|
+
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
|
126
|
+
expect(chain.normalize!).to eq({
|
127
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
128
|
+
:parameters => [[
|
129
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
130
|
+
{:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
|
131
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
132
|
+
:sub_jobs=>[
|
133
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
|
134
|
+
]
|
135
|
+
}]}
|
136
|
+
]]}
|
137
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
109
138
|
})
|
110
139
|
end
|
111
140
|
end
|
@@ -113,13 +142,18 @@ RSpec.describe CanvasSync do
|
|
113
142
|
context 'we are syncing admins without a term scope' do
|
114
143
|
it 'syncs admins separately even with no term scope' do
|
115
144
|
chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
|
116
|
-
expect(chain.
|
117
|
-
|
118
|
-
|
119
|
-
{
|
120
|
-
|
121
|
-
|
122
|
-
|
145
|
+
expect(chain.normalize!).to eq({
|
146
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
147
|
+
:parameters => [[
|
148
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
149
|
+
{:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
|
150
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
151
|
+
:sub_jobs=>[
|
152
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
|
153
|
+
]
|
154
|
+
}]}
|
155
|
+
]]}
|
156
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
123
157
|
})
|
124
158
|
end
|
125
159
|
end
|
@@ -128,13 +162,18 @@ RSpec.describe CanvasSync do
|
|
128
162
|
it "appends the SyncAssignmentsJob" do
|
129
163
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
|
130
164
|
|
131
|
-
expect(chain.
|
132
|
-
|
133
|
-
|
134
|
-
{
|
135
|
-
|
136
|
-
|
137
|
-
|
165
|
+
expect(chain.normalize!).to eq(
|
166
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
167
|
+
:parameters => [[
|
168
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
169
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
170
|
+
:sub_jobs=>[
|
171
|
+
{:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
|
172
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
173
|
+
]
|
174
|
+
}]}
|
175
|
+
]]}
|
176
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
138
177
|
)
|
139
178
|
end
|
140
179
|
end
|
@@ -143,13 +182,18 @@ RSpec.describe CanvasSync do
|
|
143
182
|
it "appends the SyncSubmissionsJob" do
|
144
183
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
|
145
184
|
|
146
|
-
expect(chain.
|
147
|
-
|
148
|
-
|
149
|
-
{
|
150
|
-
|
151
|
-
|
152
|
-
|
185
|
+
expect(chain.normalize!).to eq(
|
186
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
187
|
+
:parameters => [[
|
188
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
189
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
190
|
+
:sub_jobs=>[
|
191
|
+
{:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
|
192
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
193
|
+
]
|
194
|
+
}]}
|
195
|
+
]]}
|
196
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
153
197
|
)
|
154
198
|
end
|
155
199
|
end
|
@@ -158,95 +202,24 @@ RSpec.describe CanvasSync do
|
|
158
202
|
it "appends the SyncAssignmentGroupsJob" do
|
159
203
|
chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
|
160
204
|
|
161
|
-
expect(chain.
|
162
|
-
|
163
|
-
|
164
|
-
{
|
165
|
-
|
166
|
-
|
167
|
-
|
205
|
+
expect(chain.normalize!).to eq(
|
206
|
+
:job => "CanvasSync::Jobs::BeginSyncChainJob",
|
207
|
+
:parameters => [[
|
208
|
+
{:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
|
209
|
+
{:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
|
210
|
+
:sub_jobs=>[
|
211
|
+
{:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
|
212
|
+
{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
|
213
|
+
]
|
214
|
+
}]}
|
215
|
+
]]}
|
216
|
+
], {:legacy_support=>false, :updated_after=>nil}],
|
168
217
|
)
|
169
218
|
end
|
170
219
|
end
|
171
220
|
end
|
172
221
|
end
|
173
222
|
|
174
|
-
describe ".simple_report_sync" do
|
175
|
-
|
176
|
-
it 'invokes the first job in the queue and passes on the rest of the job chain' do
|
177
|
-
expected_job_chain = CanvasSync.simple_report_chain(
|
178
|
-
[
|
179
|
-
{
|
180
|
-
report_name: 'proservices_provisioning_csv',
|
181
|
-
model: 'users',
|
182
|
-
params: {
|
183
|
-
"parameters[include_deleted]" => true,
|
184
|
-
"parameters[users]" => true
|
185
|
-
}
|
186
|
-
}
|
187
|
-
]
|
188
|
-
).chain_data
|
189
|
-
first_job = expected_job_chain[:jobs].shift
|
190
|
-
|
191
|
-
expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
|
192
|
-
.with(
|
193
|
-
expected_job_chain,
|
194
|
-
first_job[:options]
|
195
|
-
)
|
196
|
-
|
197
|
-
CanvasSync.simple_report_sync(
|
198
|
-
[
|
199
|
-
{
|
200
|
-
report_name: 'proservices_provisioning_csv',
|
201
|
-
model: 'users',
|
202
|
-
params: {
|
203
|
-
"parameters[include_deleted]" => true,
|
204
|
-
"parameters[users]" => true
|
205
|
-
}
|
206
|
-
}
|
207
|
-
]
|
208
|
-
)
|
209
|
-
end
|
210
|
-
|
211
|
-
it 'receives the job chain for the specified table' do
|
212
|
-
chain = CanvasSync.simple_report_chain(
|
213
|
-
[
|
214
|
-
{
|
215
|
-
report_name: 'proservices_provisioning_csv',
|
216
|
-
model: 'users',
|
217
|
-
params: {
|
218
|
-
"parameters[include_deleted]" => true,
|
219
|
-
"parameters[users]" => true
|
220
|
-
}
|
221
|
-
}
|
222
|
-
]
|
223
|
-
)
|
224
|
-
|
225
|
-
expected_job_chain = {
|
226
|
-
jobs: [
|
227
|
-
{
|
228
|
-
job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
|
229
|
-
options: {
|
230
|
-
report_name: 'proservices_provisioning_csv',
|
231
|
-
model: 'users',
|
232
|
-
mapping: 'users',
|
233
|
-
klass: 'User',
|
234
|
-
term_scope: nil,
|
235
|
-
params: {
|
236
|
-
"parameters[include_deleted]" => true,
|
237
|
-
"parameters[users]" => true
|
238
|
-
}
|
239
|
-
}
|
240
|
-
}
|
241
|
-
],
|
242
|
-
global_options: {}
|
243
|
-
}
|
244
|
-
|
245
|
-
expect(chain.chain_data).to eq(expected_job_chain)
|
246
|
-
|
247
|
-
end
|
248
|
-
end
|
249
|
-
|
250
223
|
describe ".sync_scope" do
|
251
224
|
|
252
225
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
class GoodJob < CanvasSync::Job
|
4
|
-
def perform(
|
4
|
+
def perform(argument)
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
8
|
class EvilError < StandardError; end;
|
9
9
|
|
10
10
|
class BadJob < CanvasSync::Job
|
11
|
-
def perform(
|
11
|
+
def perform(argument)
|
12
12
|
raise EvilError
|
13
13
|
end
|
14
14
|
|
@@ -25,19 +25,19 @@ RSpec.describe CanvasSync::Job do
|
|
25
25
|
describe '#perform' do
|
26
26
|
it 'creates a CanvasSync::JobLog and logs relevant data on it' do
|
27
27
|
expect {
|
28
|
-
GoodJob.perform_now(
|
28
|
+
GoodJob.perform_now("argument")
|
29
29
|
}.to change { CanvasSync::JobLog.count }.by(1)
|
30
30
|
|
31
31
|
job_log = CanvasSync::JobLog.last
|
32
32
|
expect(job_log.started_at).to_not be_nil
|
33
33
|
expect(job_log.job_class).to eq(GoodJob.to_s)
|
34
|
-
expect(job_log.job_arguments).to eq([
|
34
|
+
expect(job_log.job_arguments).to eq(["argument"])
|
35
35
|
expect(job_log.completed_at).to_not be_nil
|
36
36
|
end
|
37
37
|
|
38
38
|
it 'logs exceptions on the CanvasSync::JobLog and then re-raises' do
|
39
39
|
expect {
|
40
|
-
BadJob.perform_now(
|
40
|
+
BadJob.perform_now("argument")
|
41
41
|
}.to raise_exception(StandardError)
|
42
42
|
|
43
43
|
job_log = CanvasSync::JobLog.last
|
@@ -47,25 +47,17 @@ RSpec.describe CanvasSync::Job do
|
|
47
47
|
|
48
48
|
it 'invokes an error handler' do
|
49
49
|
expect(BadJob).to receive(:good_catcher).once
|
50
|
+
set_batch_context(on_failure: 'BadJob.good_catcher')
|
50
51
|
expect {
|
51
|
-
BadJob.perform_now(
|
52
|
-
jobs: [],
|
53
|
-
global_options: {
|
54
|
-
on_failure: 'BadJob.good_catcher'
|
55
|
-
}
|
56
|
-
})
|
52
|
+
BadJob.perform_now()
|
57
53
|
}.to raise_exception(StandardError)
|
58
54
|
end
|
59
55
|
|
60
56
|
it 'logs a failing error handler' do
|
61
57
|
expect(BadJob).to receive(:bad_catcher).once.and_call_original
|
58
|
+
set_batch_context(on_failure: 'BadJob.bad_catcher')
|
62
59
|
expect {
|
63
|
-
BadJob.perform_now(
|
64
|
-
jobs: [],
|
65
|
-
global_options: {
|
66
|
-
on_failure: 'BadJob.bad_catcher'
|
67
|
-
}
|
68
|
-
})
|
60
|
+
BadJob.perform_now()
|
69
61
|
}.to raise_exception(StandardError)
|
70
62
|
job_log = CanvasSync::JobLog.last
|
71
63
|
expect(job_log.backtrace).to include "Error Occurred while handling an Error"
|