canvas_sync 0.16.5 → 0.17.0.beta5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +49 -137
  3. data/app/models/canvas_sync/sync_batch.rb +5 -0
  4. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  5. data/lib/canvas_sync.rb +35 -97
  6. data/lib/canvas_sync/importers/bulk_importer.rb +4 -7
  7. data/lib/canvas_sync/job.rb +4 -10
  8. data/lib/canvas_sync/job_batches/batch.rb +403 -0
  9. data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
  10. data/lib/canvas_sync/job_batches/callback.rb +152 -0
  11. data/lib/canvas_sync/job_batches/chain_builder.rb +220 -0
  12. data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
  13. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  14. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +19 -0
  15. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +75 -0
  16. data/lib/canvas_sync/job_batches/sidekiq.rb +93 -0
  17. data/lib/canvas_sync/job_batches/status.rb +83 -0
  18. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +35 -0
  19. data/lib/canvas_sync/jobs/report_checker.rb +3 -6
  20. data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
  21. data/lib/canvas_sync/jobs/report_starter.rb +28 -20
  22. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  23. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  24. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  25. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  26. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  27. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  28. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -34
  29. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  30. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  31. data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
  32. data/lib/canvas_sync/jobs/sync_terms_job.rb +25 -8
  33. data/lib/canvas_sync/processors/assignment_groups_processor.rb +2 -3
  34. data/lib/canvas_sync/processors/assignments_processor.rb +2 -3
  35. data/lib/canvas_sync/processors/context_module_items_processor.rb +2 -3
  36. data/lib/canvas_sync/processors/context_modules_processor.rb +2 -3
  37. data/lib/canvas_sync/processors/normal_processor.rb +1 -2
  38. data/lib/canvas_sync/processors/provisioning_report_processor.rb +2 -10
  39. data/lib/canvas_sync/processors/submissions_processor.rb +2 -3
  40. data/lib/canvas_sync/version.rb +1 -1
  41. data/spec/canvas_sync/canvas_sync_spec.rb +136 -153
  42. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  43. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  44. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  45. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  46. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  47. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  48. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  49. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  50. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  51. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
  52. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  53. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  54. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
  55. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  56. data/spec/dummy/config/environments/test.rb +2 -0
  57. data/spec/dummy/db/schema.rb +9 -1
  58. data/spec/job_batching/batch_aware_job_spec.rb +100 -0
  59. data/spec/job_batching/batch_spec.rb +372 -0
  60. data/spec/job_batching/callback_spec.rb +38 -0
  61. data/spec/job_batching/flow_spec.rb +88 -0
  62. data/spec/job_batching/integration/integration.rb +57 -0
  63. data/spec/job_batching/integration/nested.rb +88 -0
  64. data/spec/job_batching/integration/simple.rb +47 -0
  65. data/spec/job_batching/integration/workflow.rb +134 -0
  66. data/spec/job_batching/integration_helper.rb +48 -0
  67. data/spec/job_batching/sidekiq_spec.rb +124 -0
  68. data/spec/job_batching/status_spec.rb +92 -0
  69. data/spec/job_batching/support/base_job.rb +14 -0
  70. data/spec/job_batching/support/sample_callback.rb +2 -0
  71. data/spec/spec_helper.rb +17 -0
  72. metadata +85 -8
  73. data/lib/canvas_sync/job_chain.rb +0 -102
  74. data/lib/canvas_sync/jobs/fork_gather.rb +0 -74
  75. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
@@ -0,0 +1,57 @@
1
+ require 'integration_helper'
2
+
3
+ # Simple test of adding jobs to the current batch
4
+ # Batches:
5
+ # - Overall (TestWoker) + Another worker
6
+
7
+ class AnotherWorker
8
+ include Sidekiq::Worker
9
+
10
+ def perform
11
+ Sidekiq.logger.info "Another Worker"
12
+ end
13
+ end
14
+
15
+ class TestWorker
16
+ include Sidekiq::Worker
17
+
18
+ def perform
19
+ Sidekiq.logger.info "Test Worker"
20
+ if bid
21
+ batch.jobs do
22
+ AnotherWorker.perform_async
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ class MyCallback
29
+ def on_success(status, options)
30
+ Sidekiq.logger.info "Success #{options} #{status.data}"
31
+ end
32
+ alias_method :multi, :on_success
33
+
34
+ def on_complete(status, options)
35
+ Sidekiq.logger.info "Complete #{options} #{status.data}"
36
+ end
37
+ end
38
+
39
+ batch = CanvasSync::JobBatches::Batch.new
40
+ batch.description = 'Test batch'
41
+ batch.callback_queue = :default
42
+ batch.on(:success, 'MyCallback#on_success', to: 'success@gmail.com')
43
+ batch.on(:success, 'MyCallback#multi', to: 'success@gmail.com')
44
+ batch.on(:complete, MyCallback, to: 'complete@gmail.com')
45
+
46
+ batch.jobs do
47
+ 10.times do
48
+ TestWorker.perform_async
49
+ end
50
+ end
51
+ puts CanvasSync::JobBatches::Batch::Status.new(batch.bid).data
52
+
53
+ dump_redis_keys
54
+
55
+ Sidekiq::Worker.drain_all
56
+
57
+ dump_redis_keys
@@ -0,0 +1,88 @@
1
+ require 'integration_helper'
2
+
3
+ # Tests deep nesting of batches
4
+ # Batches:
5
+ # - Overall (Worker 1)
6
+ # - Worker 2
7
+ # - Worker 3
8
+ # - Worker 4
9
+
10
+ class Worker1
11
+ include Sidekiq::Worker
12
+
13
+ def perform
14
+ Sidekiq.logger.info "Work1"
15
+ batch = CanvasSync::JobBatches::Batch.new
16
+ batch.on(:success, Worker2)
17
+ batch.jobs do
18
+ Worker2.perform_async
19
+ end
20
+ end
21
+ end
22
+
23
+ class Worker2
24
+ include Sidekiq::Worker
25
+
26
+ def perform
27
+ Sidekiq.logger.info "Work2"
28
+ batch = CanvasSync::JobBatches::Batch.new
29
+ batch.on(:success, Worker3)
30
+ batch.jobs do
31
+ Worker3.perform_async
32
+ end
33
+ end
34
+
35
+ def on_success status, opts
36
+ Sidekiq.logger.info "Worker 2 Success"
37
+ end
38
+ end
39
+
40
+ class Worker3
41
+ include Sidekiq::Worker
42
+
43
+ def perform
44
+ Sidekiq.logger.info "Work3"
45
+ batch = CanvasSync::JobBatches::Batch.new
46
+ batch.on(:success, Worker4)
47
+ batch.jobs do
48
+ Worker4.perform_async
49
+ end
50
+ end
51
+
52
+ def on_success status, opts
53
+ Sidekiq.logger.info "Worker 3 Success"
54
+ end
55
+ end
56
+
57
+ class Worker4
58
+ include Sidekiq::Worker
59
+
60
+ def perform
61
+ Sidekiq.logger.info "Work4"
62
+ end
63
+
64
+ def on_success status, opts
65
+ Sidekiq.logger.info "Worker 4 Success"
66
+ end
67
+ end
68
+
69
+
70
+ class SomeClass
71
+ def on_complete(status, options)
72
+ Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
73
+ end
74
+ def on_success(status, options)
75
+ Sidekiq.logger.info "Overall Success #{options} #{status.data}"
76
+ end
77
+ end
78
+ batch = CanvasSync::JobBatches::Batch.new
79
+ batch.on(:success, SomeClass, 'uid' => 3)
80
+ batch.on(:complete, SomeClass, 'uid' => 3)
81
+ batch.jobs do
82
+ Worker1.perform_async
83
+ end
84
+
85
+ puts "Overall bid #{batch.bid}"
86
+
87
+ output, keys = process_tests
88
+ overall_tests output, keys
@@ -0,0 +1,47 @@
1
+ require 'integration_helper'
2
+
3
+ # Simple nested batch without callbacks
4
+ # Batches:
5
+ # - Overall (Worker1)
6
+ # - Worker2
7
+
8
+ class Worker1
9
+ include Sidekiq::Worker
10
+
11
+ def perform
12
+ Sidekiq.logger.info "Work1"
13
+ batch = CanvasSync::JobBatches::Batch.new
14
+ batch.jobs do
15
+ Worker2.perform_async
16
+ end
17
+ end
18
+ end
19
+
20
+ class Worker2
21
+ include Sidekiq::Worker
22
+
23
+ def perform
24
+ Sidekiq.logger.info "Work2"
25
+ end
26
+ end
27
+
28
+ class SomeClass
29
+ def on_complete(status, options)
30
+ Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
31
+ end
32
+ def on_success(status, options)
33
+ Sidekiq.logger.info "Overall Success #{options} #{status.data}"
34
+ end
35
+ end
36
+
37
+ batch = CanvasSync::JobBatches::Batch.new
38
+ batch.on(:success, SomeClass)
39
+ batch.on(:complete, SomeClass)
40
+ batch.jobs do
41
+ Worker1.perform_async
42
+ end
43
+
44
+ puts "Overall bid #{batch.bid}"
45
+
46
+ output, keys = process_tests
47
+ overall_tests output, keys
@@ -0,0 +1,134 @@
1
+ require 'integration_helper'
2
+
3
+ # Complex workflow with sequential and nested
4
+ # Also test sub batches without callbacks
5
+ # Batches:
6
+ # - Overall
7
+ # - Worker1
8
+ # - Worker3
9
+ # - Worker2 + Worker3
10
+ # - Worker1
11
+ # - Worker3
12
+ # - Worker4
13
+ # - Worker5
14
+
15
+ class Callbacks
16
+ def worker1 status, opts
17
+ Sidekiq.logger.info "Success 1 #{status.data}"
18
+
19
+ overall = CanvasSync::JobBatches::Batch.new status.parent_bid
20
+ overall.jobs do
21
+ batch = CanvasSync::JobBatches::Batch.new
22
+ batch.on(:success, "Callbacks#worker2")
23
+ batch.jobs do
24
+ Worker2.perform_async
25
+ end
26
+ end
27
+ end
28
+
29
+ def worker2 status, opts
30
+ Sidekiq.logger.info "Success 2 #{status.data}"
31
+ overall = CanvasSync::JobBatches::Batch.new status.parent_bid
32
+ overall.jobs do
33
+ batch = CanvasSync::JobBatches::Batch.new
34
+ batch.on(:success, "Callbacks#worker4")
35
+ batch.jobs do
36
+ Worker4.perform_async
37
+ end
38
+ end
39
+
40
+ end
41
+
42
+ def worker4 status, opts
43
+ Sidekiq.logger.info "Success 4 #{status.data}"
44
+ overall = CanvasSync::JobBatches::Batch.new status.parent_bid
45
+ overall.jobs do
46
+ batch = CanvasSync::JobBatches::Batch.new
47
+ batch.on(:success, "Callbacks#worker5")
48
+ batch.jobs do
49
+ Worker5.perform_async
50
+ end
51
+ end
52
+ end
53
+
54
+ def worker5 status, opts
55
+ Sidekiq.logger.info "Success 5 #{status.data}"
56
+ end
57
+ end
58
+
59
+ class Worker1
60
+ include Sidekiq::Worker
61
+
62
+ def perform
63
+ Sidekiq.logger.info "Work 1"
64
+ batch = CanvasSync::JobBatches::Batch.new
65
+ batch.jobs do
66
+ Worker3.perform_async
67
+ end
68
+ end
69
+ end
70
+
71
+ class Worker2
72
+ include Sidekiq::Worker
73
+
74
+ def perform
75
+ Sidekiq.logger.info "Work 2"
76
+ if bid
77
+ batch.jobs do
78
+ Worker3.perform_async
79
+ end
80
+ newb = CanvasSync::JobBatches::Batch.new
81
+ newb.jobs do
82
+ Worker1.perform_async
83
+ end
84
+ end
85
+ end
86
+ end
87
+
88
+ class Worker3
89
+ include Sidekiq::Worker
90
+ def perform
91
+ Sidekiq.logger.info "Work 3"
92
+ end
93
+ end
94
+
95
+ class Worker4
96
+ include Sidekiq::Worker
97
+ def perform
98
+ Sidekiq.logger.info "Work 4"
99
+ end
100
+ end
101
+
102
+ class Worker5
103
+ include Sidekiq::Worker
104
+ def perform
105
+ Sidekiq.logger.info "Work 5"
106
+ end
107
+ end
108
+
109
+ class MyCallback
110
+ def on_success(status, options)
111
+ Sidekiq.logger.info "Overall Success #{options} #{status.data}"
112
+ end
113
+ alias_method :multi, :on_success
114
+
115
+ def on_complete(status, options)
116
+ Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
117
+ end
118
+ end
119
+
120
+ overall = CanvasSync::JobBatches::Batch.new
121
+ overall.on(:success, MyCallback, to: 'success@gmail.com')
122
+ overall.on(:complete, MyCallback, to: 'success@gmail.com')
123
+ overall.jobs do
124
+ batch1 = CanvasSync::JobBatches::Batch.new
125
+ batch1.on(:success, "Callbacks#worker1")
126
+ batch1.jobs do
127
+ Worker1.perform_async
128
+ end
129
+ end
130
+
131
+ puts "Overall bid #{overall.bid}"
132
+
133
+ output, keys = process_tests
134
+ overall_tests output, keys
@@ -0,0 +1,48 @@
1
+ require 'spec_helper'
2
+ require 'sidekiq/batch'
3
+ require 'sidekiq/testing'
4
+
5
+ Sidekiq::Testing.server_middleware do |chain|
6
+ chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
7
+ end
8
+
9
+ Sidekiq.redis { |r| r.flushdb }
10
+
11
+ def redis_keys
12
+ Sidekiq.redis { |r| r.keys('BID-*') }
13
+ end
14
+
15
+ def dump_redis_keys
16
+ puts redis_keys.inspect
17
+ end
18
+
19
+ def process_tests
20
+ out_buf = StringIO.new
21
+ Sidekiq.logger = Logger.new out_buf
22
+
23
+ # Sidekiq.logger.level = :info
24
+
25
+ Sidekiq::Worker.drain_all
26
+
27
+ output = out_buf.string
28
+ keys = redis_keys
29
+ puts out_buf.string
30
+
31
+ [output, keys]
32
+ end
33
+
34
+ def overall_tests output, keys
35
+ describe "sidekiq batch" do
36
+ it "runs overall complete callback" do
37
+ expect(output).to include "Overall Complete"
38
+ end
39
+
40
+ it "runs overall success callback" do
41
+ expect(output).to include "Overall Success"
42
+ end
43
+
44
+ it "cleans redis keys" do
45
+ expect(keys).to eq([])
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,124 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe CanvasSync::JobBatches::Sidekiq do
4
+ describe CanvasSync::JobBatches::Sidekiq::ServerMiddleware do
5
+ context 'when without batch' do
6
+ it 'just yields' do
7
+ yielded = false
8
+ expect(CanvasSync::JobBatches::Batch).not_to receive(:process_successful_job)
9
+ expect(CanvasSync::JobBatches::Batch).not_to receive(:process_failed_job)
10
+ subject.call(nil, {}, nil) { yielded = true }
11
+ expect(yielded).to be_truthy
12
+ end
13
+ end
14
+
15
+ context 'when in batch' do
16
+ let(:bid) { 'SAMPLEBID' }
17
+ let(:jid) { 'SAMPLEJID' }
18
+
19
+ context 'when successful' do
20
+ it 'yields' do
21
+ yielded = false
22
+ subject.call(nil, { 'bid' => bid, 'jid' => jid }, nil) { yielded = true }
23
+ expect(yielded).to be_truthy
24
+ end
25
+
26
+ it 'calls process_successful_job' do
27
+ expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(bid, nil)
28
+ subject.call(nil, { 'bid' => bid }, nil) {}
29
+ end
30
+ end
31
+
32
+ context 'when failed' do
33
+ it 'calls process_failed_job and reraises exception' do
34
+ reraised = false
35
+ expect(CanvasSync::JobBatches::Batch).to receive(:process_failed_job)
36
+ begin
37
+ subject.call(nil, { 'bid' => bid }, nil) { raise 'ERR' }
38
+ rescue
39
+ reraised = true
40
+ end
41
+ expect(reraised).to be_truthy
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ describe CanvasSync::JobBatches::Sidekiq::ClientMiddleware do
48
+ context 'when without batch' do
49
+ it 'just yields' do
50
+ yielded = false
51
+ expect(CanvasSync::JobBatches::Batch).not_to receive(:increment_job_queue)
52
+ subject.call(nil, {}, nil) { yielded = true }
53
+ expect(yielded).to be_truthy
54
+ end
55
+ end
56
+
57
+ context 'when in batch' do
58
+ let(:bid) { 'SAMPLEBID' }
59
+ let(:jid) { 'SAMPLEJID' }
60
+ before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
61
+
62
+ it 'yields' do
63
+ yielded = false
64
+ subject.call(nil, { 'jid' => jid }, nil) { yielded = true }
65
+ expect(yielded).to be_truthy
66
+ end
67
+
68
+ it 'increments job queue' do
69
+ # expect(CanvasSync::JobBatches::Batch).to receive(:increment_job_queue).with(bid)
70
+ # subject.call(nil, { 'jid' => jid }, nil) {}
71
+ end
72
+
73
+ it 'assigns bid to msg' do
74
+ msg = { 'jid' => jid }
75
+ subject.call(nil, msg, nil) {}
76
+ expect(msg[:bid]).to eq(bid)
77
+ end
78
+ end
79
+ end
80
+ end
81
+
82
+ RSpec.describe CanvasSync::JobBatches::Sidekiq do
83
+ let(:config) { class_double(Sidekiq) }
84
+ let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
85
+
86
+ context 'client' do
87
+ it 'adds client middleware' do
88
+ expect(Sidekiq).to receive(:configure_client).and_yield(config)
89
+ expect(config).to receive(:client_middleware).and_yield(client_middleware)
90
+ expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
91
+ CanvasSync::JobBatches::Sidekiq.configure
92
+ end
93
+ end
94
+
95
+ context 'server' do
96
+ let(:server_middleware) { double(Sidekiq::Middleware::Chain) }
97
+ let(:death_handlers) { double(Array) }
98
+
99
+ it 'adds client and server middleware' do
100
+ expect(Sidekiq).to receive(:configure_server).and_yield(config)
101
+ expect(config).to receive(:client_middleware).and_yield(client_middleware)
102
+ expect(config).to receive(:server_middleware).and_yield(server_middleware)
103
+ expect(config).to receive(:death_handlers).and_return(death_handlers)
104
+ expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
105
+ expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
106
+ expect(death_handlers).to receive(:<<)
107
+ CanvasSync::JobBatches::Sidekiq.configure
108
+ end
109
+ end
110
+
111
+ context 'worker' do
112
+ it 'defines method bid' do
113
+ expect(Sidekiq::Worker.instance_methods).to include(:bid)
114
+ end
115
+
116
+ it 'defines method batch' do
117
+ expect(Sidekiq::Worker.instance_methods).to include(:batch)
118
+ end
119
+
120
+ it 'defines method valid_within_batch?' do
121
+ expect(Sidekiq::Worker.instance_methods).to include(:valid_within_batch?)
122
+ end
123
+ end
124
+ end