canvas_sync 0.18.10 → 0.19.0.beta1

data/spec/job_batching/pool_spec.rb

@@ -0,0 +1,161 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Pool do
+  include ActiveJob::TestHelper
+
+  subject { described_class.new(order: pool_order, concurrency: pool_concurrency) }
+
+  let(:pool) { subject }
+
+  let(:pool_order) { :fifo }
+  let(:pool_concurrency) { 2 }
+
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates pid when called without it' do
+      expect(subject.new.pid).not_to be_nil
+    end
+
+    it 'reuses pid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.pid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  def load_pool(count = 3)
+    jobs = count.times.map do |i|
+      { job: "BatchTestJobBase", args: [1] }
+    end
+    subject.add_jobs(jobs, skip_refill: true)
+  end
+
+  describe "#job_checked_in" do
+    it "gets called" do
+      expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).twice
+      load_pool
+      perform_enqueued_jobs do
+        pool.send :refill_allotment
+        Sidekiq::Worker.drain_all
+      end
+    end
+
+    it "refills the pool" do
+      expect(CanvasSync::JobBatches::Pool).to receive(:job_checked_in).and_call_original.exactly(3).times
+
+      load_pool
+      perform_enqueued_jobs do
+        pool.send :refill_allotment
+        Sidekiq::Worker.drain_all
+      end
+    end
+  end
+
+  describe "#cleanup_if_empty" do
+    it "cleans if pool is empty and allowed to close" do
+      expect(pool).to receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has pending" do
+      load_pool
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has active" do
+      subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has activating" do
+      subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool is empty but is kept open" do
+      pool.keep_open!
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool is empty but clean_when_empty is false" do
+      subject.redis.hset(subject.send(:redis_key), "clean_when_empty", "false")
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+  end
+
+  shared_examples "basic pool tests" do
+    describe "#push_job_to_pool" do
+      it "adds a job to the pool" do
+        subject.send(:push_job_to_pool, { job: 'job1' })
+        expect(subject.pending_count).to eq(1)
+      end
+    end
+
+    describe "#refill_allotment" do
+      it "refills the pool with jobs" do
+        load_pool
+        expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
+        subject.send(:refill_allotment)
+      end
+
+      it "limits to the concurrency count" do
+        load_pool
+        expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).twice
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 1
+      end
+
+      it "considers already active jobs against concurrency" do
+        load_pool
+        subject.redis.sadd("#{subject.send(:redis_key)}-active", "blocked")
+        expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 2
+      end
+
+      it "considers activating jobs against concurrency" do
+        load_pool
+        subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
+        expect(CanvasSync::JobBatches::ChainBuilder).to receive(:enqueue_job).once
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 2
+      end
+
+      it "doesn't fail if the pool is gone" do
+        load_pool
+        subject.cleanup_redis
+        expect(CanvasSync::JobBatches::ChainBuilder).not_to receive(:enqueue_job)
+        expect(subject.send(:refill_allotment)).to eql -1
+      end
+
+      it "doesn't fail if the pool is empty" do
+        expect(subject.send(:refill_allotment)).to eql 0
+      end
+    end
+  end
+
+  context "FIFO Pool" do
+    let(:pool_order) { :fifo }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "LIFO Pool" do
+    let(:pool_order) { :lifo }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "Random Pool" do
+    let(:pool_order) { :random }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "Priority Pool" do
+    let(:pool_order) { :priority }
+    it_behaves_like "basic pool tests"
+  end
+end

data/spec/job_batching/support/base_job.rb

@@ -1,5 +1,5 @@
 class BatchTestJobBase < ActiveJob::Base
-  def perform
+  def perform(*args, **kwargs)
   end
 
   def self.perform_async(*args)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.18.10
+  version: 0.19.0.beta1
 platform: ruby
 authors:
 - Instructure CustomDev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-06 00:00:00.000000000 Z
+date: 2023-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -485,12 +485,12 @@ files:
 - lib/canvas_sync/job_batches/chain_builder.rb
 - lib/canvas_sync/job_batches/context_hash.rb
 - lib/canvas_sync/job_batches/hier_batch_ids.lua
-- lib/canvas_sync/job_batches/hincr_max.lua
 - lib/canvas_sync/job_batches/jobs/base_job.rb
 - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
 - lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
 - lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
 - lib/canvas_sync/job_batches/pool.rb
+- lib/canvas_sync/job_batches/pool_refill.lua
 - lib/canvas_sync/job_batches/redis_model.rb
 - lib/canvas_sync/job_batches/redis_script.rb
 - lib/canvas_sync/job_batches/schedule_callback.lua
@@ -559,6 +559,7 @@ files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
+- spec/canvas_sync/misc_helper_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb
@@ -700,6 +701,7 @@ files:
 - spec/job_batching/integration/simple.rb
 - spec/job_batching/integration/workflow.rb
 - spec/job_batching/integration_helper.rb
+- spec/job_batching/pool_spec.rb
 - spec/job_batching/sidekiq_spec.rb
 - spec/job_batching/status_spec.rb
 - spec/job_batching/support/base_job.rb
@@ -745,9 +747,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.6
 signing_key: 
@@ -771,6 +773,7 @@ test_files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
+- spec/canvas_sync/misc_helper_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb
@@ -912,6 +915,7 @@ test_files:
 - spec/job_batching/integration/simple.rb
 - spec/job_batching/integration/workflow.rb
 - spec/job_batching/integration_helper.rb
+- spec/job_batching/pool_spec.rb
 - spec/job_batching/sidekiq_spec.rb
 - spec/job_batching/status_spec.rb
 - spec/job_batching/support/base_job.rb

data/lib/canvas_sync/job_batches/hincr_max.lua

@@ -1,5 +0,0 @@
-local r=redis.call('HGET', KEYS[1], ARGV[1])
-if r == false or r < ARGV[2] then
-    redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
-end
-return r or 0