canvas_sync 0.16.5 → 0.17.0.beta1

data/spec/job_batching/batch_spec.rb

@@ -0,0 +1,363 @@
+require 'spec_helper'
+
+class TestWorker < BatchTestJobBase
+  def perform
+  end
+end
+
+RSpec.describe CanvasSync::JobBatches::Batch do
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates bid when called without it' do
+      expect(subject.new.bid).not_to be_nil
+    end
+
+    it 'reuses bid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.bid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  describe '#description' do
+    let(:description) { 'custom description' }
+    before do
+      subject.description = description
+      subject.jobs { }
+    end
+
+    it 'sets descriptions' do
+      expect(subject.description).to eq(description)
+    end
+
+    it 'persists description' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'description') })
+        .to eq(description)
+    end
+  end
+
+  describe '#callback_queue' do
+    let(:callback_queue) { 'custom_queue' }
+    before do
+      subject.callback_queue = callback_queue
+      subject.jobs { }
+    end
+
+    it 'sets callback_queue' do
+      expect(subject.callback_queue).to eq(callback_queue)
+    end
+
+    it 'persists callback_queue' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'callback_queue') })
+        .to eq(callback_queue)
+    end
+  end
+
+  describe '#jobs' do
+    it 'throws error if no block given' do
+      expect { subject.jobs }.to raise_error CanvasSync::JobBatches::Batch::NoBlockGivenError
+    end
+
+    it 'increments to_process (when started)'
+
+    it 'decrements to_process (when finished)'
+    # it 'calls process_successful_job to wait for block to finish' do
+    #   batch = CanvasSync::JobBatches::Batch.new
+    #   expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(batch.bid)
+    #   batch.jobs {}
+    # end
+
+    it 'sets Thread.current bid' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do
+        expect(Thread.current[:batch]).to eq(batch)
+      end
+    end
+  end
+
+  describe '#invalidate_all' do
+    class InvalidatableJob < BatchTestJobBase
+      def perform
+        return unless valid_within_batch?
+        was_performed
+      end
+
+      def was_performed; end
+    end
+
+    it 'marks batch in redis as invalidated' do
+      batch = CanvasSync::JobBatches::Batch.new
+      job = InvalidatableJob.new
+      allow(job).to receive(:was_performed)
+
+      batch.invalidate_all
+      batch.jobs { job.perform }
+
+      expect(job).not_to have_received(:was_performed)
+    end
+
+    context 'nested batches' do
+      let(:batch_parent) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_1) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_2) { CanvasSync::JobBatches::Batch.new }
+      let(:job_of_parent) { InvalidatableJob.new }
+      let(:job_of_child_1) { InvalidatableJob.new }
+      let(:job_of_child_2) { InvalidatableJob.new }
+
+      before do
+        allow(job_of_parent).to receive(:was_performed)
+        allow(job_of_child_1).to receive(:was_performed)
+        allow(job_of_child_2).to receive(:was_performed)
+      end
+
+      it 'invalidates all job if parent batch is marked as invalidated' do
+        batch_parent.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).not_to have_received(:was_performed)
+        expect(job_of_child_1).not_to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+
+      it 'invalidates only requested batch' do
+        batch_child_2.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).to have_received(:was_performed)
+        expect(job_of_child_1).to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+    end
+  end
+
+  describe '#process_failed_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      let(:failed_jid) { 'xxx' }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+      end
+
+      it 'add job to failed list' do
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+        failed = CanvasSync::JobBatches::Batch.redis { |r| r.smembers("BID-#{bid}-failed") }
+        expect(failed).to eq(['xxx', 'failed-job-id'])
+      end
+    end
+  end
+
+  describe '#process_successful_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      before { batch.on(:complete, Object) }
+      # before { batch.increment_job_queue(bid) }
+      before { batch.jobs do TestWorker.perform_async end }
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, 'failed-job-id')
+      end
+    end
+
+    context 'success' do
+      before { batch.on(:complete, Object) }
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'cleanups redis key' do
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+        expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
+      end
+    end
+  end
+
+  describe '#increment_job_queue' do
+    let(:bid) { 'BID' }
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+
+    it 'increments pending' do
+      batch.jobs do TestWorker.perform_async end
+      pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
+      expect(pending).to eq('1')
+    end
+
+    it 'increments total' do
+      batch.jobs do TestWorker.perform_async end
+      total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
+      expect(total).to eq('1')
+    end
+  end
+
+  describe '#enqueue_callbacks' do
+    let(:callback) { double('callback') }
+    let(:event) { :complete }
+
+    context 'on :success' do
+      let(:event) { :success }
+      context 'when no callbacks are defined' do
+        it 'clears redis keys' do
+          batch = CanvasSync::JobBatches::Batch.new
+          expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
+          CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+        end
+      end
+    end
+
+    context 'when already called' do
+      it 'returns and does not enqueue callbacks' do
+        batch = CanvasSync::JobBatches::Batch.new
+        batch.on(event, SampleCallback)
+        CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{batch.bid}", event, true) }
+
+        expect(batch).not_to receive(:push_callbacks)
+        CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+      end
+    end
+
+    context 'With ActiveJob Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback2', event.to_s, opts2, batch.bid, nil
+            )
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+      end
+    end
+
+    context 'With Sidekiq Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [
+                ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
+                ['SampleCallback', event.to_s, opts, batch.bid, nil]
+              ],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/job_batching/callback_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Batch::Callback::Worker do
+  describe '#perform' do
+    it 'does not do anything if it cannot find the callback class' do
+      subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'does not do anything if event is different from complete or success' do
+      expect(SampleCallback).not_to receive(:new)
+      subject.perform('SampleCallback', 'ups', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls on_success if defined' do
+      callback_instance = double('SampleCallback', on_success: true)
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:on_success)
+        .with(instance_of(CanvasSync::JobBatches::Batch::Status), {})
+      subject.perform('SampleCallback', 'success', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls on_complete if defined' do
+      callback_instance = double('SampleCallback')
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:on_complete)
+        .with(instance_of(CanvasSync::JobBatches::Batch::Status), {})
+      subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls specific callback if defined' do
+      callback_instance = double('SampleCallback')
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:sample_method)
+        .with(instance_of(CanvasSync::JobBatches::Batch::Status), {})
+      subject.perform('SampleCallback#sample_method', 'complete', {}, 'ABCD', 'EFGH')
+    end
+  end
+end

data/spec/job_batching/flow_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+class WorkerA < BatchTestJobBase
+  def perform
+    puts 'A'
+  end
+end
+
+class WorkerB < BatchTestJobBase
+  def perform
+    puts 'B'
+  end
+end
+
+class WorkerC < BatchTestJobBase
+  def perform
+    puts 'C'
+  end
+end
+
+RSpec.describe 'Batch flow' do
+  context 'when handling a batch' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    before { batch.on(:complete, SampleCallback, :id => 42) }
+    before { batch.description = 'describing the batch' }
+    let(:status) { CanvasSync::JobBatches::Batch::Status.new(batch.bid) }
+    let(:jids) { batch.jobs do 3.times do TestWorker.perform_async end end }
+    let(:queue) { Sidekiq::Queue.new }
+
+    it 'correctly initializes' do
+      expect(jids.size).to eq(3)
+
+      expect(batch.bid).not_to be_nil
+      expect(batch.description).to eq('describing the batch')
+
+      batch.jobs {}
+
+      expect(status.total).to eq(3)
+      expect(status.pending).to eq(3)
+      expect(status.failures).to eq(0)
+      expect(status.complete?).to be false
+      expect(status.created_at).not_to be_nil
+      expect(status.bid).to eq(batch.bid)
+    end
+
+    it 'handles an empty batch' do
+      batch = CanvasSync::JobBatches::Batch.new
+      jids = batch.jobs do nil end
+      expect(jids.size).to eq(0)
+    end
+  end
+
+  context 'when handling a nested batch' do
+    let(:batchA) { CanvasSync::JobBatches::Batch.new }
+    let(:batchB) { CanvasSync::JobBatches::Batch.new }
+    let(:batchC) { CanvasSync::JobBatches::Batch.new(batchA.bid) }
+    let(:batchD) { CanvasSync::JobBatches::Batch.new }
+    let(:jids) { [] }
+    let(:parent) { batchA.bid }
+    let(:children) { [] }
+
+    it 'handles a basic nested batch' do
+      batchA.jobs do
+        jids << WorkerA.perform_async
+        batchB.jobs do
+          jids << WorkerB.perform_async
+        end
+        jids << WorkerA.perform_async
+        children << batchB.bid
+      end
+
+      batchC.jobs do
+        batchD.jobs do
+          jids << WorkerC.perform_async
+        end
+        children << batchD.bid
+      end
+
+      expect(jids.size).to eq(4)
+      expect(CanvasSync::JobBatches::Batch::Status.new(parent).child_count).to eq(2)
+      children.each do |kid|
+          status = CanvasSync::JobBatches::Batch::Status.new(kid)
+          expect(status.child_count).to eq(0)
+          expect(status.pending).to eq(1)
+          expect(status.parent_bid).to eq(parent)
+      end
+
+    end
+
+  end
+end

data/spec/job_batching/integration/integration.rb

@@ -0,0 +1,57 @@
+require 'integration_helper'
+
+# Simple test of adding jobs to the current batch
+# Batches:
+# - Overall (TestWoker) + Another worker
+
+class AnotherWorker
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Another Worker"
+  end
+end
+
+class TestWorker
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Test Worker"
+    if bid
+      batch.jobs do
+        AnotherWorker.perform_async
+      end
+    end
+  end
+end
+
+class MyCallback
+  def on_success(status, options)
+    Sidekiq.logger.info "Success #{options} #{status.data}"
+  end
+  alias_method :multi, :on_success
+
+  def on_complete(status, options)
+    Sidekiq.logger.info "Complete #{options} #{status.data}"
+  end
+end
+
+batch = CanvasSync::JobBatches::Batch.new
+batch.description = 'Test batch'
+batch.callback_queue = :default
+batch.on(:success, 'MyCallback#on_success', to: 'success@gmail.com')
+batch.on(:success, 'MyCallback#multi', to: 'success@gmail.com')
+batch.on(:complete, MyCallback, to: 'complete@gmail.com')
+
+batch.jobs do
+  10.times do
+    TestWorker.perform_async
+  end
+end
+puts CanvasSync::JobBatches::Batch::Status.new(batch.bid).data
+
+dump_redis_keys
+
+Sidekiq::Worker.drain_all
+
+dump_redis_keys

data/spec/job_batching/integration/nested.rb

@@ -0,0 +1,88 @@
+require 'integration_helper'
+
+# Tests deep nesting of batches
+# Batches:
+# - Overall (Worker 1)
+#  - Worker 2
+#   - Worker 3
+#    - Worker 4
+
+class Worker1
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work1"
+    batch = CanvasSync::JobBatches::Batch.new
+    batch.on(:success, Worker2)
+    batch.jobs do
+      Worker2.perform_async
+    end
+  end
+end
+
+class Worker2
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work2"
+    batch = CanvasSync::JobBatches::Batch.new
+    batch.on(:success, Worker3)
+    batch.jobs do
+      Worker3.perform_async
+    end
+  end
+
+  def on_success status, opts
+    Sidekiq.logger.info "Worker 2 Success"
+  end
+end
+
+class Worker3
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work3"
+    batch = CanvasSync::JobBatches::Batch.new
+    batch.on(:success, Worker4)
+    batch.jobs do
+      Worker4.perform_async
+    end
+  end
+
+  def on_success status, opts
+    Sidekiq.logger.info "Worker 3 Success"
+  end
+end
+
+class Worker4
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work4"
+  end
+
+  def on_success status, opts
+    Sidekiq.logger.info "Worker 4 Success"
+  end
+end
+
+
+class SomeClass
+  def on_complete(status, options)
+    Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
+  end
+  def on_success(status, options)
+    Sidekiq.logger.info "Overall Success #{options} #{status.data}"
+  end
+end
+batch = CanvasSync::JobBatches::Batch.new
+batch.on(:success, SomeClass, 'uid' => 3)
+batch.on(:complete, SomeClass, 'uid' => 3)
+batch.jobs do
+  Worker1.perform_async
+end
+
+puts "Overall bid #{batch.bid}"
+
+output, keys = process_tests
+overall_tests output, keys

data/spec/job_batching/integration/simple.rb

@@ -0,0 +1,47 @@
+require 'integration_helper'
+
+# Simple nested batch without callbacks
+# Batches:
+# - Overall (Worker1)
+#  - Worker2
+
+class Worker1
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work1"
+    batch = CanvasSync::JobBatches::Batch.new
+    batch.jobs do
+      Worker2.perform_async
+    end
+  end
+end
+
+class Worker2
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work2"
+  end
+end
+
+class SomeClass
+  def on_complete(status, options)
+    Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
+  end
+  def on_success(status, options)
+    Sidekiq.logger.info "Overall Success #{options} #{status.data}"
+  end
+end
+
+batch = CanvasSync::JobBatches::Batch.new
+batch.on(:success, SomeClass)
+batch.on(:complete, SomeClass)
+batch.jobs do
+  Worker1.perform_async
+end
+
+puts "Overall bid #{batch.bid}"
+
+output, keys = process_tests
+overall_tests output, keys