canvas_sync 0.16.2 → 0.17.0.beta3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -137
- data/app/models/canvas_sync/sync_batch.rb +5 -0
- data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
- data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
- data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
- data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
- data/lib/canvas_sync.rb +36 -118
- data/lib/canvas_sync/concerns/api_syncable.rb +27 -0
- data/lib/canvas_sync/job.rb +5 -5
- data/lib/canvas_sync/job_batches/batch.rb +399 -0
- data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
- data/lib/canvas_sync/job_batches/callback.rb +153 -0
- data/lib/canvas_sync/job_batches/chain_builder.rb +210 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
- data/lib/canvas_sync/job_batches/sidekiq.rb +93 -0
- data/lib/canvas_sync/job_batches/status.rb +63 -0
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
- data/lib/canvas_sync/jobs/report_checker.rb +3 -6
- data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
- data/lib/canvas_sync/jobs/report_starter.rb +27 -19
- data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
- data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +5 -35
- data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
- data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
- data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
- data/lib/canvas_sync/jobs/sync_terms_job.rb +25 -8
- data/lib/canvas_sync/misc_helper.rb +15 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/spec/canvas_sync/canvas_sync_spec.rb +136 -153
- data/spec/canvas_sync/jobs/job_spec.rb +9 -17
- data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
- data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
- data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
- data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
- data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
- data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
- data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
- data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
- data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
- data/spec/dummy/app/models/account.rb +3 -0
- data/spec/dummy/app/models/pseudonym.rb +14 -0
- data/spec/dummy/app/models/submission.rb +1 -0
- data/spec/dummy/app/models/user.rb +1 -0
- data/spec/dummy/config/environments/test.rb +2 -0
- data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
- data/spec/dummy/db/schema.rb +24 -4
- data/spec/job_batching/batch_aware_job_spec.rb +100 -0
- data/spec/job_batching/batch_spec.rb +363 -0
- data/spec/job_batching/callback_spec.rb +38 -0
- data/spec/job_batching/flow_spec.rb +91 -0
- data/spec/job_batching/integration/integration.rb +57 -0
- data/spec/job_batching/integration/nested.rb +88 -0
- data/spec/job_batching/integration/simple.rb +47 -0
- data/spec/job_batching/integration/workflow.rb +134 -0
- data/spec/job_batching/integration_helper.rb +48 -0
- data/spec/job_batching/sidekiq_spec.rb +124 -0
- data/spec/job_batching/status_spec.rb +92 -0
- data/spec/job_batching/support/base_job.rb +14 -0
- data/spec/job_batching/support/sample_callback.rb +2 -0
- data/spec/spec_helper.rb +17 -0
- metadata +90 -8
- data/lib/canvas_sync/job_chain.rb +0 -57
- data/lib/canvas_sync/jobs/fork_gather.rb +0 -59
- data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
@@ -1,57 +0,0 @@
|
|
1
|
-
module CanvasSync
|
2
|
-
class JobChain
|
3
|
-
attr_reader :chain_data
|
4
|
-
|
5
|
-
delegate_missing_to :chain_data
|
6
|
-
|
7
|
-
VALID_PLACEMENT_PARAMETERS = %i[before after].freeze
|
8
|
-
|
9
|
-
def initialize(chain_data)
|
10
|
-
@chain_data = chain_data
|
11
|
-
end
|
12
|
-
|
13
|
-
def jobs
|
14
|
-
chain_data[:jobs]
|
15
|
-
end
|
16
|
-
|
17
|
-
def global_options
|
18
|
-
chain_data[:global_options]
|
19
|
-
end
|
20
|
-
|
21
|
-
def merge_options(job, options)
|
22
|
-
jobs.each do |j|
|
23
|
-
j[:options].deep_merge!(options) if job_matches_pattern(j, job)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
def insert(new_job, **kwargs)
|
28
|
-
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
29
|
-
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
30
|
-
raise "Exactly one placement parameter may be provided" if kwargs.values.compact!.length > 1
|
31
|
-
|
32
|
-
if !kwargs.present?
|
33
|
-
jobs << new_job
|
34
|
-
else
|
35
|
-
placement = kwargs.keys[0]
|
36
|
-
relative_to = kwargs.values[0]
|
37
|
-
|
38
|
-
index = jobs.index { |job| job_matches_pattern(job, relative_to) }
|
39
|
-
raise "Could not find a \"#{relative_to}\" job in the chain" if index.nil?
|
40
|
-
|
41
|
-
index += 1 if placement == :after
|
42
|
-
new_job[:job] = new_job[:job].to_s
|
43
|
-
jobs.insert(index, new_job)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def process!(extra_options: {})
|
48
|
-
CanvasSync::invoke_next(self, extra_options: extra_options)
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
|
53
|
-
def job_matches_pattern(job_entry, pattern)
|
54
|
-
job_entry[:job].to_s == pattern.to_s
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,59 +0,0 @@
|
|
1
|
-
module CanvasSync
|
2
|
-
module Jobs
|
3
|
-
class ForkGather < CanvasSync::Job
|
4
|
-
def perform(job_chain, options)
|
5
|
-
forked_job = self.class.forked_at_job(job_chain)
|
6
|
-
|
7
|
-
if forked_job.present?
|
8
|
-
forked_job.with_lock do
|
9
|
-
forked_job.fork_count -= 1
|
10
|
-
forked_job.save!
|
11
|
-
end
|
12
|
-
|
13
|
-
if forked_job.fork_count <= 0
|
14
|
-
(job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
|
15
|
-
job_chain[:global_options].delete(k.to_sym)
|
16
|
-
end
|
17
|
-
CanvasSync.invoke_next(job_chain)
|
18
|
-
end
|
19
|
-
else
|
20
|
-
CanvasSync.invoke_next(job_chain)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
|
25
|
-
return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
|
26
|
-
|
27
|
-
duped_chain = CanvasSync.duplicate_chain(job_chain)
|
28
|
-
job_list = duped_chain[:jobs]
|
29
|
-
while job_list.count > 0
|
30
|
-
job_class_name = job_list[0][:job]
|
31
|
-
job_class = job_class_name.constantize
|
32
|
-
break if job_class <= CanvasSync::Jobs::ForkGather
|
33
|
-
job_list.shift
|
34
|
-
end
|
35
|
-
|
36
|
-
return nil unless job_list.present?
|
37
|
-
|
38
|
-
if skip_invoke
|
39
|
-
duped_chain
|
40
|
-
else
|
41
|
-
CanvasSync.invoke_next(duped_chain)
|
42
|
-
true
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
protected
|
47
|
-
|
48
|
-
def self.forked_at_job(job_chain)
|
49
|
-
fork_item = (job_chain[:global_options][:fork_path] || []).pop
|
50
|
-
|
51
|
-
if fork_item.present?
|
52
|
-
CanvasSync::JobLog.find_by(job_id: fork_item)
|
53
|
-
else
|
54
|
-
nil
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
@@ -1,73 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
RSpec.describe CanvasSync::Jobs::ForkGather do
|
4
|
-
describe '#perform' do
|
5
|
-
let!(:job_log) { CanvasSync::JobLog.create!(job_id: 'BLAH', fork_count: 3) }
|
6
|
-
let(:job_chain) { {jobs: [], global_options: { fork_path: ['BLAH'] }} }
|
7
|
-
|
8
|
-
it 'decrements fork_count' do
|
9
|
-
CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
|
10
|
-
expect(job_log.reload.fork_count).to eq 2
|
11
|
-
end
|
12
|
-
|
13
|
-
it 'does not continue chain if fork_count > 0' do
|
14
|
-
expect(CanvasSync).not_to receive(:invoke_next)
|
15
|
-
CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'continues the chain if fork_count <= 0' do
|
19
|
-
job_log.update!(fork_count: 1)
|
20
|
-
expect(CanvasSync).to receive(:invoke_next)
|
21
|
-
CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'continues if no fork_path is specified' do
|
25
|
-
job_chain[:global_options].delete(:fork_path)
|
26
|
-
expect(CanvasSync).to receive(:invoke_next)
|
27
|
-
CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
|
28
|
-
end
|
29
|
-
|
30
|
-
it 'pops the most-recent fork_path enrty' do
|
31
|
-
job_log.update!(fork_count: 1)
|
32
|
-
expect(CanvasSync).to receive(:invoke_next) do |*args|
|
33
|
-
expect(args[0][:global_options][:fork_path]).to eq []
|
34
|
-
end
|
35
|
-
CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
describe 'handle_branch_error' do
|
40
|
-
let(:error) { StandardError.new }
|
41
|
-
|
42
|
-
let(:job_chain) {
|
43
|
-
{
|
44
|
-
jobs: [
|
45
|
-
{ job: 'CanvasSync::Jobs::ReportChecker' },
|
46
|
-
{ job: 'CanvasSync::Jobs::ReportChecker' },
|
47
|
-
{ job: 'CanvasSync::Jobs::ForkGather' },
|
48
|
-
{ job: 'CanvasSync::Jobs::ReportChecker' },
|
49
|
-
],
|
50
|
-
global_options: {
|
51
|
-
fork_path: ['BLAH'],
|
52
|
-
}
|
53
|
-
}
|
54
|
-
}
|
55
|
-
|
56
|
-
it 'skips to and performs the next ForkGatherJob' do
|
57
|
-
expect(CanvasSync).to receive(:invoke_next) do |*args|
|
58
|
-
expect(args[0][:jobs][0][:job]).to eq 'CanvasSync::Jobs::ForkGather'
|
59
|
-
end
|
60
|
-
expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be true
|
61
|
-
end
|
62
|
-
|
63
|
-
it 'does nothing if no ForkGather is in the chain' do
|
64
|
-
job_chain[:jobs].delete_at(2)
|
65
|
-
expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
|
66
|
-
end
|
67
|
-
|
68
|
-
it 'does nothing if no fork_path is present' do
|
69
|
-
job_chain[:global_options][:fork_path] = []
|
70
|
-
expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|