canvas_sync 0.16.4 → 0.17.0.beta4

data/spec/job_batching/batch_aware_job_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::BatchAwareJob do
+  include ActiveJob::TestHelper
+
+  after do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+
+  context "When Performing" do
+    context 'when without batch' do
+      it 'just yields' do
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_successful_job)
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_failed_job)
+        expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+
+        BatchTestJobBase.perform_now
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+
+      context 'when successful' do
+        it 'yields' do
+          expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+          BatchTestJobBase.perform_now
+        end
+
+        it 'calls process_successful_job' do
+          job = BatchTestJobBase.new
+          job.instance_variable_set(:@bid, bid)
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(bid, job.job_id)
+          job.perform_now
+        end
+      end
+
+      context 'when failed' do
+        it 'calls process_failed_job and reraises exception' do
+          reraised = false
+          job = FailingBatchTestJobBase.new
+          job.instance_variable_set(:@bid, bid)
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_failed_job)
+          begin
+            job.perform_now
+          rescue
+            reraised = true
+          end
+          expect(reraised).to be_truthy
+        end
+      end
+    end
+  end
+
+  context "When Enqueueing" do
+    context 'when without batch' do
+      it 'just yields' do
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:increment_job_queue)
+        BatchTestJobBase.perform_later
+        expect(BatchTestJobBase).to have_been_enqueued
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+
+      before do
+        Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid)
+        Thread.current[:batch].instance_variable_set(:@open, true)
+      end
+
+      it 'yields' do
+        expect {
+          BatchTestJobBase.perform_later
+        }.to enqueue_job(BatchTestJobBase)
+      end
+
+      it 'assigns bid to job metadata' do
+        job = BatchTestJobBase.perform_later
+        expect(job.bid).to eq bid
+        expect(job.serialize['batch_id']).to eq bid
+      end
+    end
+  end
+
+  context 'worker' do
+    it 'defines method bid' do
+      expect(ActiveJob::Base.instance_methods).to include(:bid)
+    end
+
+    it 'defines method batch' do
+      expect(ActiveJob::Base.instance_methods).to include(:batch)
+    end
+
+    it 'defines method valid_within_batch?' do
+      expect(ActiveJob::Base.instance_methods).to include(:valid_within_batch?)
+    end
+  end
+end

data/spec/job_batching/batch_spec.rb

@@ -0,0 +1,372 @@
+require 'spec_helper'
+
+class TestWorker < BatchTestJobBase
+  def perform
+  end
+end
+
+RSpec.describe CanvasSync::JobBatches::Batch do
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates bid when called without it' do
+      expect(subject.new.bid).not_to be_nil
+    end
+
+    it 'reuses bid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.bid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  describe '#description' do
+    let(:description) { 'custom description' }
+    before do
+      subject.description = description
+      subject.jobs { }
+    end
+
+    it 'sets descriptions' do
+      expect(subject.description).to eq(description)
+    end
+
+    it 'persists description' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'description') })
+        .to eq(description)
+    end
+  end
+
+  describe '#callback_queue' do
+    let(:callback_queue) { 'custom_queue' }
+    before do
+      subject.callback_queue = callback_queue
+      subject.jobs { }
+    end
+
+    it 'sets callback_queue' do
+      expect(subject.callback_queue).to eq(callback_queue)
+    end
+
+    it 'persists callback_queue' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'callback_queue') })
+        .to eq(callback_queue)
+    end
+  end
+
+  describe '#jobs' do
+    it 'throws error if no block given' do
+      expect { subject.jobs }.to raise_error CanvasSync::JobBatches::Batch::NoBlockGivenError
+    end
+
+    it 'increments to_process (when started)'
+
+    it 'decrements to_process (when finished)'
+    # it 'calls process_successful_job to wait for block to finish' do
+    #   batch = CanvasSync::JobBatches::Batch.new
+    #   expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(batch.bid)
+    #   batch.jobs {}
+    # end
+
+    it 'sets Thread.current bid' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do
+        expect(Thread.current[:batch]).to eq(batch)
+      end
+    end
+  end
+
+  describe '#invalidate_all' do
+    class InvalidatableJob < BatchTestJobBase
+      def perform
+        return unless valid_within_batch?
+        was_performed
+      end
+
+      def was_performed; end
+    end
+
+    it 'marks batch in redis as invalidated' do
+      batch = CanvasSync::JobBatches::Batch.new
+      job = InvalidatableJob.new
+      allow(job).to receive(:was_performed)
+
+      batch.invalidate_all
+      batch.jobs { job.perform }
+
+      expect(job).not_to have_received(:was_performed)
+    end
+
+    context 'nested batches' do
+      let(:batch_parent) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_1) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_2) { CanvasSync::JobBatches::Batch.new }
+      let(:job_of_parent) { InvalidatableJob.new }
+      let(:job_of_child_1) { InvalidatableJob.new }
+      let(:job_of_child_2) { InvalidatableJob.new }
+
+      before do
+        allow(job_of_parent).to receive(:was_performed)
+        allow(job_of_child_1).to receive(:was_performed)
+        allow(job_of_child_2).to receive(:was_performed)
+      end
+
+      it 'invalidates all job if parent batch is marked as invalidated' do
+        batch_parent.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).not_to have_received(:was_performed)
+        expect(job_of_child_1).not_to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+
+      it 'invalidates only requested batch' do
+        batch_child_2.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).to have_received(:was_performed)
+        expect(job_of_child_1).to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+    end
+  end
+
+  describe '#process_failed_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      let(:failed_jid) { 'xxx' }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+      end
+
+      it 'add job to failed list' do
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+        failed = CanvasSync::JobBatches::Batch.redis { |r| r.smembers("BID-#{bid}-failed") }
+        expect(failed).to eq(['xxx', 'failed-job-id'])
+      end
+    end
+  end
+
+  describe '#process_successful_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      before { batch.on(:complete, Object) }
+      # before { batch.increment_job_queue(bid) }
+      # before { batch.jobs do TestWorker.perform_async end }
+      # before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
+      end
+    end
+
+    context 'success' do
+      before { batch.on(:complete, Object) }
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'tries to call success callback after a previous failure' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
+
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'cleanups redis key' do
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+        expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
+      end
+    end
+  end
+
+  describe '#increment_job_queue' do
+    let(:bid) { 'BID' }
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+
+    it 'increments pending' do
+      batch.jobs do TestWorker.perform_async end
+      pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
+      expect(pending).to eq('1')
+    end
+
+    it 'increments total' do
+      batch.jobs do TestWorker.perform_async end
+      total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
+      expect(total).to eq('1')
+    end
+  end
+
+  describe '#enqueue_callbacks' do
+    let(:callback) { double('callback') }
+    let(:event) { :complete }
+
+    context 'on :success' do
+      let(:event) { :success }
+      context 'when no callbacks are defined' do
+        it 'clears redis keys' do
+          batch = CanvasSync::JobBatches::Batch.new
+          expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
+          CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+        end
+      end
+    end
+
+    context 'when already called' do
+      it 'returns and does not enqueue callbacks' do
+        batch = CanvasSync::JobBatches::Batch.new
+        batch.on(event, SampleCallback)
+        CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{batch.bid}", event, true) }
+
+        expect(batch).not_to receive(:push_callbacks)
+        CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+      end
+    end
+
+    context 'With ActiveJob Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback2', event.to_s, opts2, batch.bid, nil
+            )
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+      end
+    end
+
+    context 'With Sidekiq Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [
+                ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
+                ['SampleCallback', event.to_s, opts, batch.bid, nil]
+              ],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+      end
+    end
+  end
+end