canvas_sync 0.18.12 → 0.19.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe CanvasSync::JobBatches::Pool do
4
+ include ActiveJob::TestHelper
5
+
6
+ subject { described_class.new(order: pool_order, concurrency: pool_concurrency) }
7
+
8
+ let(:pool) { subject }
9
+
10
+ let(:pool_order) { :fifo }
11
+ let(:pool_concurrency) { 2 }
12
+
13
+ describe '#initialize' do
14
+ subject { described_class }
15
+
16
+ it 'creates pid when called without it' do
17
+ expect(subject.new.pid).not_to be_nil
18
+ end
19
+
20
+ it 'reuses pid when called with it' do
21
+ batch = subject.new('dayPO5KxuRXXxw')
22
+ expect(batch.pid).to eq('dayPO5KxuRXXxw')
23
+ end
24
+ end
25
+
26
+ def load_pool(count = 3)
27
+ jobs = count.times.map do |i|
28
+ { job: "BatchTestJobBase", args: [1] }
29
+ end
30
+ subject.add_jobs(jobs, skip_refill: true)
31
+ end
32
+
33
+ describe "#job_checked_in" do
34
+ it "gets called" do
35
+ expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).twice
36
+ load_pool
37
+ perform_enqueued_jobs do
38
+ pool.send :refill_allotment
39
+ Sidekiq::Worker.drain_all
40
+ end
41
+ end
42
+
43
+ it "refills the pool" do
44
+ expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).and_call_original.exactly(3).times
45
+
46
+ load_pool
47
+ perform_enqueued_jobs do
48
+ pool.send :refill_allotment
49
+ Sidekiq::Worker.drain_all
50
+ end
51
+ end
52
+ end
53
+
54
+ describe "#cleanup_if_empty" do
55
+ it "cleans if pool is empty and allowed to close" do
56
+ expect(pool).to receive(:cleanup_redis)
57
+ pool.cleanup_if_empty
58
+ end
59
+
60
+ it "doesn't clean if pool has pending" do
61
+ load_pool
62
+ expect(pool).to_not receive(:cleanup_redis)
63
+ pool.cleanup_if_empty
64
+ end
65
+
66
+ it "doesn't clean if pool has active" do
67
+ subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
68
+ expect(pool).to_not receive(:cleanup_redis)
69
+ pool.cleanup_if_empty
70
+ end
71
+
72
+ it "doesn't clean if pool has activating" do
73
+ subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
74
+ expect(pool).to_not receive(:cleanup_redis)
75
+ pool.cleanup_if_empty
76
+ end
77
+
78
+ it "doesn't clean if pool is empty but is kept open" do
79
+ pool.keep_open!
80
+ expect(pool).to_not receive(:cleanup_redis)
81
+ pool.cleanup_if_empty
82
+ end
83
+
84
+ it "doesn't clean if pool is empty but clean_when_empty is false" do
85
+ subject.redis.hset(subject.send(:redis_key), "clean_when_empty", "false")
86
+ expect(pool).to_not receive(:cleanup_redis)
87
+ pool.cleanup_if_empty
88
+ end
89
+ end
90
+
91
+ shared_examples "basic pool tests" do
92
+ describe "#push_job_to_pool" do
93
+ it "adds a job to the pool" do
94
+ subject.send(:push_job_to_pool, { job: 'job1' })
95
+ expect(subject.pending_count).to eq(1)
96
+ end
97
+ end
98
+
99
+ describe "#refill_allotment" do
100
+ it "refills the pool with jobs" do
101
+ load_pool
102
+ expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
103
+ subject.send(:refill_allotment)
104
+ end
105
+
106
+ it "limits to the concurrency count" do
107
+ load_pool
108
+ expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
109
+ expect(subject.send(:refill_allotment)).to eql 2
110
+ expect(subject.pending_count).to eql 1
111
+ end
112
+
113
+ it "considers already active jobs against concurrency" do
114
+ load_pool
115
+ subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
116
+ expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
117
+ expect(subject.send(:refill_allotment)).to eql 2
118
+ expect(subject.pending_count).to eql 2
119
+ end
120
+
121
+ it "considers activating jobs against concurrency" do
122
+ load_pool
123
+ subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
124
+ expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
125
+ expect(subject.send(:refill_allotment)).to eql 2
126
+ expect(subject.pending_count).to eql 2
127
+ end
128
+
129
+ it "doesn't fail if the pool is gone" do
130
+ load_pool
131
+ subject.cleanup_redis
132
+ expect(CanvasSync::JobBatches::ChainBuilder).not_to receive(:enqueue_job)
133
+ expect(subject.send(:refill_allotment)).to eql -1
134
+ end
135
+
136
+ it "doesn't fail if the pool is empty" do
137
+ expect(subject.send(:refill_allotment)).to eql 0
138
+ end
139
+ end
140
+ end
141
+
142
+ context "FIFO Pool" do
143
+ let(:pool_order) { :fifo }
144
+ it_behaves_like "basic pool tests"
145
+ end
146
+
147
+ context "LIFO Pool" do
148
+ let(:pool_order) { :lifo }
149
+ it_behaves_like "basic pool tests"
150
+ end
151
+
152
+ context "Random Pool" do
153
+ let(:pool_order) { :random }
154
+ it_behaves_like "basic pool tests"
155
+ end
156
+
157
+ context "Priority Pool" do
158
+ let(:pool_order) { :priority }
159
+ it_behaves_like "basic pool tests"
160
+ end
161
+ end
@@ -1,5 +1,5 @@
1
1
  class BatchTestJobBase < ActiveJob::Base
2
- def perform
2
+ def perform(*args, **kwargs)
3
3
  end
4
4
 
5
5
  def self.perform_async(*args)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.18.12
4
+ version: 0.19.0.beta2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Instructure CustomDev
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-10-27 00:00:00.000000000 Z
11
+ date: 2023-11-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -413,6 +413,7 @@ files:
413
413
  - lib/canvas_sync/class_callback_executor.rb
414
414
  - lib/canvas_sync/concerns/ability_helper.rb
415
415
  - lib/canvas_sync/concerns/account/ancestry.rb
416
+ - lib/canvas_sync/concerns/account/base.rb
416
417
  - lib/canvas_sync/concerns/api_syncable.rb
417
418
  - lib/canvas_sync/concerns/auto_relations.rb
418
419
  - lib/canvas_sync/concerns/legacy_columns.rb
@@ -485,12 +486,12 @@ files:
485
486
  - lib/canvas_sync/job_batches/chain_builder.rb
486
487
  - lib/canvas_sync/job_batches/context_hash.rb
487
488
  - lib/canvas_sync/job_batches/hier_batch_ids.lua
488
- - lib/canvas_sync/job_batches/hincr_max.lua
489
489
  - lib/canvas_sync/job_batches/jobs/base_job.rb
490
490
  - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
491
491
  - lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
492
492
  - lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
493
493
  - lib/canvas_sync/job_batches/pool.rb
494
+ - lib/canvas_sync/job_batches/pool_refill.lua
494
495
  - lib/canvas_sync/job_batches/redis_model.rb
495
496
  - lib/canvas_sync/job_batches/redis_script.rb
496
497
  - lib/canvas_sync/job_batches/schedule_callback.lua
@@ -559,6 +560,7 @@ files:
559
560
  - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
560
561
  - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
561
562
  - spec/canvas_sync/jobs/sync_terms_job_spec.rb
563
+ - spec/canvas_sync/misc_helper_spec.rb
562
564
  - spec/canvas_sync/models/accounts_spec.rb
563
565
  - spec/canvas_sync/models/admins_spec.rb
564
566
  - spec/canvas_sync/models/assignment_group_spec.rb
@@ -700,6 +702,7 @@ files:
700
702
  - spec/job_batching/integration/simple.rb
701
703
  - spec/job_batching/integration/workflow.rb
702
704
  - spec/job_batching/integration_helper.rb
705
+ - spec/job_batching/pool_spec.rb
703
706
  - spec/job_batching/sidekiq_spec.rb
704
707
  - spec/job_batching/status_spec.rb
705
708
  - spec/job_batching/support/base_job.rb
@@ -745,9 +748,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
745
748
  version: '0'
746
749
  required_rubygems_version: !ruby/object:Gem::Requirement
747
750
  requirements:
748
- - - ">="
751
+ - - ">"
749
752
  - !ruby/object:Gem::Version
750
- version: '0'
753
+ version: 1.3.1
751
754
  requirements: []
752
755
  rubygems_version: 3.1.6
753
756
  signing_key:
@@ -771,6 +774,7 @@ test_files:
771
774
  - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
772
775
  - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
773
776
  - spec/canvas_sync/jobs/sync_terms_job_spec.rb
777
+ - spec/canvas_sync/misc_helper_spec.rb
774
778
  - spec/canvas_sync/models/accounts_spec.rb
775
779
  - spec/canvas_sync/models/admins_spec.rb
776
780
  - spec/canvas_sync/models/assignment_group_spec.rb
@@ -912,6 +916,7 @@ test_files:
912
916
  - spec/job_batching/integration/simple.rb
913
917
  - spec/job_batching/integration/workflow.rb
914
918
  - spec/job_batching/integration_helper.rb
919
+ - spec/job_batching/pool_spec.rb
915
920
  - spec/job_batching/sidekiq_spec.rb
916
921
  - spec/job_batching/status_spec.rb
917
922
  - spec/job_batching/support/base_job.rb
@@ -1,5 +0,0 @@
1
- local r=redis.call('HGET', KEYS[1], ARGV[1])
2
- if r == false or r < ARGV[2] then
3
- redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
4
- end
5
- return r or 0