canvas_sync 0.18.12 → 0.19.0.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -9
- data/lib/canvas_sync/concerns/ability_helper.rb +19 -11
- data/lib/canvas_sync/concerns/account/base.rb +15 -0
- data/lib/canvas_sync/job_batches/batch.rb +46 -2
- data/lib/canvas_sync/job_batches/callback.rb +7 -4
- data/lib/canvas_sync/job_batches/chain_builder.rb +38 -74
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +5 -5
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +54 -22
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +5 -5
- data/lib/canvas_sync/job_batches/pool.rb +83 -54
- data/lib/canvas_sync/job_batches/pool_refill.lua +40 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/_jobs_table.erb +2 -0
- data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb +10 -1
- data/lib/canvas_sync/job_batches/sidekiq/web.rb +3 -1
- data/lib/canvas_sync/jobs/canvas_process_waiter.rb +3 -35
- data/lib/canvas_sync/misc_helper.rb +48 -0
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +2 -9
- data/spec/canvas_sync/canvas_sync_spec.rb +201 -115
- data/spec/canvas_sync/jobs/canvas_process_waiter_spec.rb +0 -48
- data/spec/canvas_sync/misc_helper_spec.rb +58 -0
- data/spec/dummy/log/test.log +69092 -0
- data/spec/job_batching/pool_spec.rb +161 -0
- data/spec/job_batching/support/base_job.rb +1 -1
- metadata +10 -5
- data/lib/canvas_sync/job_batches/hincr_max.lua +0 -5
@@ -0,0 +1,161 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe CanvasSync::JobBatches::Pool do
|
4
|
+
include ActiveJob::TestHelper
|
5
|
+
|
6
|
+
subject { described_class.new(order: pool_order, concurrency: pool_concurrency) }
|
7
|
+
|
8
|
+
let(:pool) { subject }
|
9
|
+
|
10
|
+
let(:pool_order) { :fifo }
|
11
|
+
let(:pool_concurrency) { 2 }
|
12
|
+
|
13
|
+
describe '#initialize' do
|
14
|
+
subject { described_class }
|
15
|
+
|
16
|
+
it 'creates pid when called without it' do
|
17
|
+
expect(subject.new.pid).not_to be_nil
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'reuses pid when called with it' do
|
21
|
+
batch = subject.new('dayPO5KxuRXXxw')
|
22
|
+
expect(batch.pid).to eq('dayPO5KxuRXXxw')
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def load_pool(count = 3)
|
27
|
+
jobs = count.times.map do |i|
|
28
|
+
{ job: "BatchTestJobBase", args: [1] }
|
29
|
+
end
|
30
|
+
subject.add_jobs(jobs, skip_refill: true)
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "#job_checked_in" do
|
34
|
+
it "gets called" do
|
35
|
+
expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).twice
|
36
|
+
load_pool
|
37
|
+
perform_enqueued_jobs do
|
38
|
+
pool.send :refill_allotment
|
39
|
+
Sidekiq::Worker.drain_all
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
it "refills the pool" do
|
44
|
+
expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).and_call_original.exactly(3).times
|
45
|
+
|
46
|
+
load_pool
|
47
|
+
perform_enqueued_jobs do
|
48
|
+
pool.send :refill_allotment
|
49
|
+
Sidekiq::Worker.drain_all
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe "#cleanup_if_empty" do
|
55
|
+
it "cleans if pool is empty and allowed to close" do
|
56
|
+
expect(pool).to receive(:cleanup_redis)
|
57
|
+
pool.cleanup_if_empty
|
58
|
+
end
|
59
|
+
|
60
|
+
it "doesn't clean if pool has pending" do
|
61
|
+
load_pool
|
62
|
+
expect(pool).to_not receive(:cleanup_redis)
|
63
|
+
pool.cleanup_if_empty
|
64
|
+
end
|
65
|
+
|
66
|
+
it "doesn't clean if pool has active" do
|
67
|
+
subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
|
68
|
+
expect(pool).to_not receive(:cleanup_redis)
|
69
|
+
pool.cleanup_if_empty
|
70
|
+
end
|
71
|
+
|
72
|
+
it "doesn't clean if pool has activating" do
|
73
|
+
subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
|
74
|
+
expect(pool).to_not receive(:cleanup_redis)
|
75
|
+
pool.cleanup_if_empty
|
76
|
+
end
|
77
|
+
|
78
|
+
it "doesn't clean if pool is empty but is kept open" do
|
79
|
+
pool.keep_open!
|
80
|
+
expect(pool).to_not receive(:cleanup_redis)
|
81
|
+
pool.cleanup_if_empty
|
82
|
+
end
|
83
|
+
|
84
|
+
it "doesn't clean if pool is empty but clean_when_empty is false" do
|
85
|
+
subject.redis.hset(subject.send(:redis_key), "clean_when_empty", "false")
|
86
|
+
expect(pool).to_not receive(:cleanup_redis)
|
87
|
+
pool.cleanup_if_empty
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
shared_examples "basic pool tests" do
|
92
|
+
describe "#push_job_to_pool" do
|
93
|
+
it "adds a job to the pool" do
|
94
|
+
subject.send(:push_job_to_pool, { job: 'job1' })
|
95
|
+
expect(subject.pending_count).to eq(1)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
describe "#refill_allotment" do
|
100
|
+
it "refills the pool with jobs" do
|
101
|
+
load_pool
|
102
|
+
expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
|
103
|
+
subject.send(:refill_allotment)
|
104
|
+
end
|
105
|
+
|
106
|
+
it "limits to the concurrency count" do
|
107
|
+
load_pool
|
108
|
+
expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
|
109
|
+
expect(subject.send(:refill_allotment)).to eql 2
|
110
|
+
expect(subject.pending_count).to eql 1
|
111
|
+
end
|
112
|
+
|
113
|
+
it "considers already active jobs against concurrency" do
|
114
|
+
load_pool
|
115
|
+
subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
|
116
|
+
expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
|
117
|
+
expect(subject.send(:refill_allotment)).to eql 2
|
118
|
+
expect(subject.pending_count).to eql 2
|
119
|
+
end
|
120
|
+
|
121
|
+
it "considers activating jobs against concurrency" do
|
122
|
+
load_pool
|
123
|
+
subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
|
124
|
+
expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
|
125
|
+
expect(subject.send(:refill_allotment)).to eql 2
|
126
|
+
expect(subject.pending_count).to eql 2
|
127
|
+
end
|
128
|
+
|
129
|
+
it "doesn't fail if the pool is gone" do
|
130
|
+
load_pool
|
131
|
+
subject.cleanup_redis
|
132
|
+
expect(CanvasSync::JobBatches::ChainBuilder).not_to receive(:enqueue_job)
|
133
|
+
expect(subject.send(:refill_allotment)).to eql -1
|
134
|
+
end
|
135
|
+
|
136
|
+
it "doesn't fail if the pool is empty" do
|
137
|
+
expect(subject.send(:refill_allotment)).to eql 0
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context "FIFO Pool" do
|
143
|
+
let(:pool_order) { :fifo }
|
144
|
+
it_behaves_like "basic pool tests"
|
145
|
+
end
|
146
|
+
|
147
|
+
context "LIFO Pool" do
|
148
|
+
let(:pool_order) { :lifo }
|
149
|
+
it_behaves_like "basic pool tests"
|
150
|
+
end
|
151
|
+
|
152
|
+
context "Random Pool" do
|
153
|
+
let(:pool_order) { :random }
|
154
|
+
it_behaves_like "basic pool tests"
|
155
|
+
end
|
156
|
+
|
157
|
+
context "Priority Pool" do
|
158
|
+
let(:pool_order) { :priority }
|
159
|
+
it_behaves_like "basic pool tests"
|
160
|
+
end
|
161
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: canvas_sync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.19.0.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Instructure CustomDev
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -413,6 +413,7 @@ files:
|
|
413
413
|
- lib/canvas_sync/class_callback_executor.rb
|
414
414
|
- lib/canvas_sync/concerns/ability_helper.rb
|
415
415
|
- lib/canvas_sync/concerns/account/ancestry.rb
|
416
|
+
- lib/canvas_sync/concerns/account/base.rb
|
416
417
|
- lib/canvas_sync/concerns/api_syncable.rb
|
417
418
|
- lib/canvas_sync/concerns/auto_relations.rb
|
418
419
|
- lib/canvas_sync/concerns/legacy_columns.rb
|
@@ -485,12 +486,12 @@ files:
|
|
485
486
|
- lib/canvas_sync/job_batches/chain_builder.rb
|
486
487
|
- lib/canvas_sync/job_batches/context_hash.rb
|
487
488
|
- lib/canvas_sync/job_batches/hier_batch_ids.lua
|
488
|
-
- lib/canvas_sync/job_batches/hincr_max.lua
|
489
489
|
- lib/canvas_sync/job_batches/jobs/base_job.rb
|
490
490
|
- lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
|
491
491
|
- lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
|
492
492
|
- lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
|
493
493
|
- lib/canvas_sync/job_batches/pool.rb
|
494
|
+
- lib/canvas_sync/job_batches/pool_refill.lua
|
494
495
|
- lib/canvas_sync/job_batches/redis_model.rb
|
495
496
|
- lib/canvas_sync/job_batches/redis_script.rb
|
496
497
|
- lib/canvas_sync/job_batches/schedule_callback.lua
|
@@ -559,6 +560,7 @@ files:
|
|
559
560
|
- spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
|
560
561
|
- spec/canvas_sync/jobs/sync_submissions_job_spec.rb
|
561
562
|
- spec/canvas_sync/jobs/sync_terms_job_spec.rb
|
563
|
+
- spec/canvas_sync/misc_helper_spec.rb
|
562
564
|
- spec/canvas_sync/models/accounts_spec.rb
|
563
565
|
- spec/canvas_sync/models/admins_spec.rb
|
564
566
|
- spec/canvas_sync/models/assignment_group_spec.rb
|
@@ -700,6 +702,7 @@ files:
|
|
700
702
|
- spec/job_batching/integration/simple.rb
|
701
703
|
- spec/job_batching/integration/workflow.rb
|
702
704
|
- spec/job_batching/integration_helper.rb
|
705
|
+
- spec/job_batching/pool_spec.rb
|
703
706
|
- spec/job_batching/sidekiq_spec.rb
|
704
707
|
- spec/job_batching/status_spec.rb
|
705
708
|
- spec/job_batching/support/base_job.rb
|
@@ -745,9 +748,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
745
748
|
version: '0'
|
746
749
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
747
750
|
requirements:
|
748
|
-
- - "
|
751
|
+
- - ">"
|
749
752
|
- !ruby/object:Gem::Version
|
750
|
-
version:
|
753
|
+
version: 1.3.1
|
751
754
|
requirements: []
|
752
755
|
rubygems_version: 3.1.6
|
753
756
|
signing_key:
|
@@ -771,6 +774,7 @@ test_files:
|
|
771
774
|
- spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
|
772
775
|
- spec/canvas_sync/jobs/sync_submissions_job_spec.rb
|
773
776
|
- spec/canvas_sync/jobs/sync_terms_job_spec.rb
|
777
|
+
- spec/canvas_sync/misc_helper_spec.rb
|
774
778
|
- spec/canvas_sync/models/accounts_spec.rb
|
775
779
|
- spec/canvas_sync/models/admins_spec.rb
|
776
780
|
- spec/canvas_sync/models/assignment_group_spec.rb
|
@@ -912,6 +916,7 @@ test_files:
|
|
912
916
|
- spec/job_batching/integration/simple.rb
|
913
917
|
- spec/job_batching/integration/workflow.rb
|
914
918
|
- spec/job_batching/integration_helper.rb
|
919
|
+
- spec/job_batching/pool_spec.rb
|
915
920
|
- spec/job_batching/sidekiq_spec.rb
|
916
921
|
- spec/job_batching/status_spec.rb
|
917
922
|
- spec/job_batching/support/base_job.rb
|