canvas_sync 0.16.2 → 0.17.0.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +49 -137
  3. data/app/models/canvas_sync/sync_batch.rb +5 -0
  4. data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
  5. data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
  6. data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
  7. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  8. data/lib/canvas_sync.rb +36 -118
  9. data/lib/canvas_sync/concerns/api_syncable.rb +27 -0
  10. data/lib/canvas_sync/job.rb +5 -5
  11. data/lib/canvas_sync/job_batches/batch.rb +399 -0
  12. data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
  13. data/lib/canvas_sync/job_batches/callback.rb +153 -0
  14. data/lib/canvas_sync/job_batches/chain_builder.rb +210 -0
  15. data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
  16. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  17. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
  18. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
  19. data/lib/canvas_sync/job_batches/sidekiq.rb +93 -0
  20. data/lib/canvas_sync/job_batches/status.rb +63 -0
  21. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
  22. data/lib/canvas_sync/jobs/report_checker.rb +3 -6
  23. data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
  24. data/lib/canvas_sync/jobs/report_starter.rb +27 -19
  25. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  26. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  27. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  28. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  29. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  30. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  31. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +5 -35
  32. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  33. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  34. data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
  35. data/lib/canvas_sync/jobs/sync_terms_job.rb +25 -8
  36. data/lib/canvas_sync/misc_helper.rb +15 -0
  37. data/lib/canvas_sync/version.rb +1 -1
  38. data/spec/canvas_sync/canvas_sync_spec.rb +136 -153
  39. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  40. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  41. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  42. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  43. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  44. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  45. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  46. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  47. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  48. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
  49. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  50. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  51. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
  52. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  53. data/spec/dummy/app/models/account.rb +3 -0
  54. data/spec/dummy/app/models/pseudonym.rb +14 -0
  55. data/spec/dummy/app/models/submission.rb +1 -0
  56. data/spec/dummy/app/models/user.rb +1 -0
  57. data/spec/dummy/config/environments/test.rb +2 -0
  58. data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
  59. data/spec/dummy/db/schema.rb +24 -4
  60. data/spec/job_batching/batch_aware_job_spec.rb +100 -0
  61. data/spec/job_batching/batch_spec.rb +363 -0
  62. data/spec/job_batching/callback_spec.rb +38 -0
  63. data/spec/job_batching/flow_spec.rb +91 -0
  64. data/spec/job_batching/integration/integration.rb +57 -0
  65. data/spec/job_batching/integration/nested.rb +88 -0
  66. data/spec/job_batching/integration/simple.rb +47 -0
  67. data/spec/job_batching/integration/workflow.rb +134 -0
  68. data/spec/job_batching/integration_helper.rb +48 -0
  69. data/spec/job_batching/sidekiq_spec.rb +124 -0
  70. data/spec/job_batching/status_spec.rb +92 -0
  71. data/spec/job_batching/support/base_job.rb +14 -0
  72. data/spec/job_batching/support/sample_callback.rb +2 -0
  73. data/spec/spec_helper.rb +17 -0
  74. metadata +90 -8
  75. data/lib/canvas_sync/job_chain.rb +0 -57
  76. data/lib/canvas_sync/jobs/fork_gather.rb +0 -59
  77. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
@@ -1,57 +0,0 @@
1
- module CanvasSync
2
- class JobChain
3
- attr_reader :chain_data
4
-
5
- delegate_missing_to :chain_data
6
-
7
- VALID_PLACEMENT_PARAMETERS = %i[before after].freeze
8
-
9
- def initialize(chain_data)
10
- @chain_data = chain_data
11
- end
12
-
13
- def jobs
14
- chain_data[:jobs]
15
- end
16
-
17
- def global_options
18
- chain_data[:global_options]
19
- end
20
-
21
- def merge_options(job, options)
22
- jobs.each do |j|
23
- j[:options].deep_merge!(options) if job_matches_pattern(j, job)
24
- end
25
- end
26
-
27
- def insert(new_job, **kwargs)
28
- invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
29
- raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
30
- raise "Exactly one placement parameter may be provided" if kwargs.values.compact!.length > 1
31
-
32
- if !kwargs.present?
33
- jobs << new_job
34
- else
35
- placement = kwargs.keys[0]
36
- relative_to = kwargs.values[0]
37
-
38
- index = jobs.index { |job| job_matches_pattern(job, relative_to) }
39
- raise "Could not find a \"#{relative_to}\" job in the chain" if index.nil?
40
-
41
- index += 1 if placement == :after
42
- new_job[:job] = new_job[:job].to_s
43
- jobs.insert(index, new_job)
44
- end
45
- end
46
-
47
- def process!(extra_options: {})
48
- CanvasSync::invoke_next(self, extra_options: extra_options)
49
- end
50
-
51
- private
52
-
53
- def job_matches_pattern(job_entry, pattern)
54
- job_entry[:job].to_s == pattern.to_s
55
- end
56
- end
57
- end
@@ -1,59 +0,0 @@
1
- module CanvasSync
2
- module Jobs
3
- class ForkGather < CanvasSync::Job
4
- def perform(job_chain, options)
5
- forked_job = self.class.forked_at_job(job_chain)
6
-
7
- if forked_job.present?
8
- forked_job.with_lock do
9
- forked_job.fork_count -= 1
10
- forked_job.save!
11
- end
12
-
13
- if forked_job.fork_count <= 0
14
- (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
15
- job_chain[:global_options].delete(k.to_sym)
16
- end
17
- CanvasSync.invoke_next(job_chain)
18
- end
19
- else
20
- CanvasSync.invoke_next(job_chain)
21
- end
22
- end
23
-
24
- def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
25
- return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
26
-
27
- duped_chain = CanvasSync.duplicate_chain(job_chain)
28
- job_list = duped_chain[:jobs]
29
- while job_list.count > 0
30
- job_class_name = job_list[0][:job]
31
- job_class = job_class_name.constantize
32
- break if job_class <= CanvasSync::Jobs::ForkGather
33
- job_list.shift
34
- end
35
-
36
- return nil unless job_list.present?
37
-
38
- if skip_invoke
39
- duped_chain
40
- else
41
- CanvasSync.invoke_next(duped_chain)
42
- true
43
- end
44
- end
45
-
46
- protected
47
-
48
- def self.forked_at_job(job_chain)
49
- fork_item = (job_chain[:global_options][:fork_path] || []).pop
50
-
51
- if fork_item.present?
52
- CanvasSync::JobLog.find_by(job_id: fork_item)
53
- else
54
- nil
55
- end
56
- end
57
- end
58
- end
59
- end
@@ -1,73 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe CanvasSync::Jobs::ForkGather do
4
- describe '#perform' do
5
- let!(:job_log) { CanvasSync::JobLog.create!(job_id: 'BLAH', fork_count: 3) }
6
- let(:job_chain) { {jobs: [], global_options: { fork_path: ['BLAH'] }} }
7
-
8
- it 'decrements fork_count' do
9
- CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
10
- expect(job_log.reload.fork_count).to eq 2
11
- end
12
-
13
- it 'does not continue chain if fork_count > 0' do
14
- expect(CanvasSync).not_to receive(:invoke_next)
15
- CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
16
- end
17
-
18
- it 'continues the chain if fork_count <= 0' do
19
- job_log.update!(fork_count: 1)
20
- expect(CanvasSync).to receive(:invoke_next)
21
- CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
22
- end
23
-
24
- it 'continues if no fork_path is specified' do
25
- job_chain[:global_options].delete(:fork_path)
26
- expect(CanvasSync).to receive(:invoke_next)
27
- CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
28
- end
29
-
30
- it 'pops the most-recent fork_path enrty' do
31
- job_log.update!(fork_count: 1)
32
- expect(CanvasSync).to receive(:invoke_next) do |*args|
33
- expect(args[0][:global_options][:fork_path]).to eq []
34
- end
35
- CanvasSync::Jobs::ForkGather.perform_now(job_chain, {})
36
- end
37
- end
38
-
39
- describe 'handle_branch_error' do
40
- let(:error) { StandardError.new }
41
-
42
- let(:job_chain) {
43
- {
44
- jobs: [
45
- { job: 'CanvasSync::Jobs::ReportChecker' },
46
- { job: 'CanvasSync::Jobs::ReportChecker' },
47
- { job: 'CanvasSync::Jobs::ForkGather' },
48
- { job: 'CanvasSync::Jobs::ReportChecker' },
49
- ],
50
- global_options: {
51
- fork_path: ['BLAH'],
52
- }
53
- }
54
- }
55
-
56
- it 'skips to and performs the next ForkGatherJob' do
57
- expect(CanvasSync).to receive(:invoke_next) do |*args|
58
- expect(args[0][:jobs][0][:job]).to eq 'CanvasSync::Jobs::ForkGather'
59
- end
60
- expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be true
61
- end
62
-
63
- it 'does nothing if no ForkGather is in the chain' do
64
- job_chain[:jobs].delete_at(2)
65
- expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
66
- end
67
-
68
- it 'does nothing if no fork_path is present' do
69
- job_chain[:global_options][:fork_path] = []
70
- expect(CanvasSync::Jobs::ForkGather.handle_branch_error(error, job_chain: job_chain)).to be nil
71
- end
72
- end
73
- end