joblin 0.1.0

data/spec/batching/batch_spec.rb

@@ -0,0 +1,493 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::Batching::Batch do
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates bid when called without it' do
+      expect(subject.new.bid).not_to be_nil
+    end
+
+    it 'reuses bid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.bid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  describe '#description' do
+    let(:description) { 'custom description' }
+    before do
+      subject.description = description
+      subject.jobs { }
+    end
+
+    it 'sets descriptions' do
+      expect(subject.description).to eq(description)
+    end
+
+    it 'persists description' do
+      expect(Joblin::Batching::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'description') })
+        .to eq(description)
+    end
+  end
+
+  describe '#callback_queue' do
+    let(:callback_queue) { 'custom_queue' }
+    before do
+      subject.callback_queue = callback_queue
+      subject.jobs { }
+    end
+
+    it 'sets callback_queue' do
+      expect(subject.callback_queue).to eq(callback_queue)
+    end
+
+    it 'persists callback_queue' do
+      expect(Joblin::Batching::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'callback_queue') })
+        .to eq(callback_queue)
+    end
+  end
+
+  describe '#jobs' do
+    it 'throws error if no block given' do
+      expect { subject.jobs }.to raise_error Joblin::Batching::Batch::NoBlockGivenError
+    end
+
+    it 'increments to_process (when started)'
+
+    it 'decrements to_process (when finished)'
+    # it 'calls process_successful_job to wait for block to finish' do
+    #   batch = Joblin::Batching::Batch.new
+    #   expect(Joblin::Batching::Batch).to receive(:process_successful_job).with(batch.bid)
+    #   batch.jobs {}
+    # end
+
+    it 'sets Thread.current bid' do
+      batch = Joblin::Batching::Batch.new
+      batch.jobs do
+        expect(Thread.current[Joblin::Batching::CURRENT_BATCH_THREAD_KEY]).to eq(batch)
+      end
+    end
+  end
+
+  describe '#invalidate_all' do
+    class InvalidatableJob < BatchTestJobBase
+      def perform
+        return unless valid_within_batch?
+        was_performed
+      end
+
+      def was_performed; end
+    end
+
+    it 'marks batch in redis as invalidated' do
+      batch = Joblin::Batching::Batch.new
+      job = InvalidatableJob.new
+      allow(job).to receive(:was_performed)
+
+      batch.invalidate_all
+      batch.jobs { job.perform }
+
+      expect(job).not_to have_received(:was_performed)
+    end
+
+    context 'nested batches' do
+      let(:batch_parent) { Joblin::Batching::Batch.new }
+      let(:batch_child_1) { Joblin::Batching::Batch.new }
+      let(:batch_child_2) { Joblin::Batching::Batch.new }
+      let(:job_of_parent) { InvalidatableJob.new }
+      let(:job_of_child_1) { InvalidatableJob.new }
+      let(:job_of_child_2) { InvalidatableJob.new }
+
+      before do
+        allow(job_of_parent).to receive(:was_performed)
+        allow(job_of_child_1).to receive(:was_performed)
+        allow(job_of_child_2).to receive(:was_performed)
+      end
+
+      it 'invalidates all job if parent batch is marked as invalidated' do
+        batch_parent.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).not_to have_received(:was_performed)
+        expect(job_of_child_1).not_to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+
+      it 'invalidates only requested batch' do
+        batch_child_2.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).to have_received(:was_performed)
+        expect(job_of_child_1).to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+    end
+  end
+
+  describe '#process_failed_job' do
+    let(:batch) { Joblin::Batching::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      let(:failed_jid) { 'xxx' }
+
+      it 'tries to call complete callback' do
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        Joblin::Batching::Batch.process_failed_job(bid, failed_jid)
+      end
+
+      it 'add job to failed list' do
+        Joblin::Batching::Batch.process_failed_job(bid, 'failed-job-id')
+        Joblin::Batching::Batch.process_failed_job(bid, failed_jid)
+        failed = Joblin::Batching::Batch.redis { |r| r.smembers("BID-#{bid}-failed") }
+        expect(failed).to contain_exactly('xxx', 'failed-job-id')
+      end
+    end
+  end
+
+  describe '#process_dead_job' do
+    let(:batch) { Joblin::Batching::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'dead' do
+      let(:failed_jid) { 'xxx' }
+
+      it 'tries to call death callback' do
+        allow(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:death, bid)
+        Joblin::Batching::Batch.process_dead_job(bid, failed_jid)
+      end
+
+      it 'add job to failed list' do
+        Joblin::Batching::Batch.process_dead_job(bid, 'failed-job-id')
+        Joblin::Batching::Batch.process_dead_job(bid, failed_jid)
+        failed = Joblin::Batching::Batch.redis { |r| r.smembers("BID-#{bid}-dead") }
+        expect(failed).to contain_exactly('xxx', 'failed-job-id')
+      end
+
+      it 'automattically triggers the :stagnated callback' do
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
+        allow(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:death, bid)
+        Joblin::Batching::Batch.process_dead_job(bid, failed_jid)
+      end
+
+      it 'does not trigger :stagnated if pending jobs are still present' do
+        Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
+        expect(Joblin::Batching::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
+        Joblin::Batching::Batch.process_dead_job(bid, failed_jid)
+      end
+
+      it 'does not trigger :stagnated if pending batches are still present' do
+        Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
+        expect(Joblin::Batching::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
+        Joblin::Batching::Batch.process_dead_job(bid, failed_jid)
+      end
+    end
+  end
+
+  describe '#process_successful_job' do
+    let(:batch) { Joblin::Batching::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      before { batch.on(:complete, Object) }
+      # before { batch.append_jobs(bid) }
+      # before { batch.jobs do BatchTestWorker.perform_async end }
+      # before { Joblin::Batching::Batch.process_failed_job(bid, 'failed-job-id') }
+
+      it 'tries to call complete callback' do
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        Joblin::Batching::Batch.process_failed_job(bid, 'failed-job-id')
+      end
+    end
+
+    context 'success' do
+      before { batch.on(:complete, Object) }
+
+      it 'tries to call complete callback' do
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        Joblin::Batching::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'tries to call success callback after a previous failure' do
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        Joblin::Batching::Batch.process_failed_job(bid, jid)
+
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(Joblin::Batching::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        Joblin::Batching::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        Joblin::Batching::Batch::Callback.worker_class = Joblin::Batching::Compat::Sidekiq::SidekiqCallbackWorker
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        Sidekiq::Testing.inline! do
+          Joblin::Batching::Batch.process_failed_job(bid, jid)
+          Joblin::Batching::Batch.process_successful_job(bid, jid)
+        end
+      end
+
+      it 'delays triggering callbacks if keep_open is set' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        Joblin::Batching::Batch::Callback.worker_class = Joblin::Batching::Compat::Sidekiq::SidekiqCallbackWorker
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).not_to receive(:on_complete)
+        expect(callback_instance).not_to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+        batch.keep_open!
+
+        Sidekiq::Testing.inline! do
+          Joblin::Batching::Batch.process_failed_job(bid, jid)
+          Joblin::Batching::Batch.process_successful_job(bid, jid)
+        end
+
+        RSpec::Mocks.space.proxy_for(callback_instance).reset
+
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        Sidekiq::Testing.inline! do
+          batch.let_close!
+        end
+      end
+
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        Joblin::Batching::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 0) }
+
+        class RetryingJob < BatchTestJobBase
+          @@failed = false
+
+          def perform
+            unless @@failed
+              @@failed = true
+              raise "A Failure"
+            end
+          end
+        end
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        batch.jobs do
+          RetryingJob.perform_later
+        end
+
+        job_def = Sidekiq::Worker.jobs[0]
+        int_job_class = job_def["class"].constantize
+
+        begin
+          int_job_class.process_job(job_def)
+        rescue
+        end
+        Sidekiq::Worker.drain_all
+      end
+
+      it 'cleanups redis key' do
+        Joblin::Batching::Batch.process_successful_job(bid, jid)
+        expect(Joblin::Batching::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
+      end
+    end
+  end
+
+  describe '#append_jobs' do
+    let(:batch) { Joblin::Batching::Batch.new }
+
+    it 'increments pending' do
+      batch.jobs do BatchTestWorker.perform_async end
+      pending = Joblin::Batching::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
+      expect(pending).to eq('1')
+    end
+  end
+
+  describe '#enqueue_callbacks' do
+    let(:callback) { double('callback') }
+    let(:event) { :complete }
+
+    context 'on :success' do
+      let(:event) { :success }
+      context 'when no callbacks are defined' do
+        it 'clears redis keys' do
+          batch = Joblin::Batching::Batch.new
+          batch.jobs {}
+          expect(Joblin::Batching::Batch).to receive(:cleanup_redis).with(batch.bid)
+          Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+        end
+      end
+    end
+
+    context 'when already called' do
+      it 'returns and does not enqueue callbacks' do
+        batch = Joblin::Batching::Batch.new
+        batch.on(event, SampleCallback)
+        Joblin::Batching::Batch.redis { |r| r.hset("BID-#{batch.bid}", event, "true") }
+
+        expect(batch).not_to receive(:push_callbacks)
+        Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+      end
+    end
+
+    context 'With ActiveJob Adapter' do
+      around(:all) do |block|
+        Joblin::Batching::Batch::Callback.worker_class = Joblin::Batching::Compat::ActiveJob::ActiveJobCallbackWorker
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = Joblin::Batching::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            ActiveJob::Base.queue_adapter = :test
+
+            batch = Joblin::Batching::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.jobs {}
+
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+
+            expect(Joblin::Batching::Batch::Callback.worker_class).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            ActiveJob::Base.queue_adapter = :test
+
+            batch = Joblin::Batching::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
+
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+            expect(Joblin::Batching::Batch::Callback.worker_class).to have_been_enqueued.with(
+              'SampleCallback2', event.to_s, opts2, batch.bid, nil
+            )
+            expect(Joblin::Batching::Batch::Callback.worker_class).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+      end
+    end
+
+    context 'With Sidekiq Adapter' do
+      around(:all) do |block|
+        Joblin::Batching::Batch::Callback.worker_class = Joblin::Batching::Compat::Sidekiq::SidekiqCallbackWorker
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = Joblin::Batching::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = Joblin::Batching::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.jobs{}
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback.worker_class,
+              'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
+              'queue' => 'default'
+            )
+
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = Joblin::Batching::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback.worker_class,
+              'args' => array_including(
+                ['SampleCallback', event.to_s, opts, batch.bid, nil],
+                ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
+              ),
+              'queue' => 'default'
+            )
+
+            Joblin::Batching::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/batching/callback_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::Batching::Batch::Callback.worker_class do
+  describe '#perform' do
+    it 'does not do anything if it cannot find the callback class' do
+      subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'does not do anything if event is different from complete or success' do
+      expect(SampleCallback).not_to receive(:new)
+      subject.perform('SampleCallback', 'ups', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls on_success if defined' do
+      callback_instance = double('SampleCallback', on_success: true)
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:on_success)
+        .with(instance_of(Joblin::Batching::Batch::Status), {})
+      subject.perform('SampleCallback', 'success', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls on_complete if defined' do
+      callback_instance = double('SampleCallback')
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:on_complete)
+        .with(instance_of(Joblin::Batching::Batch::Status), {})
+      subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')
+    end
+
+    it 'calls specific callback if defined' do
+      callback_instance = double('SampleCallback')
+      expect(SampleCallback).to receive(:new).and_return(callback_instance)
+      expect(callback_instance).to receive(:sample_method)
+        .with(instance_of(Joblin::Batching::Batch::Status), {})
+      subject.perform('SampleCallback#sample_method', 'complete', {}, 'ABCD', 'EFGH')
+    end
+  end
+end

data/spec/batching/compat/active_job_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::Batching::Compat::ActiveJob do
+  describe Joblin::Batching::Compat::ActiveJob::BatchAwareJob do
+    include ActiveJob::TestHelper
+
+    after do
+      clear_enqueued_jobs
+      clear_performed_jobs
+    end
+
+    context "When Performing" do
+      context 'when without batch' do
+        it 'just yields' do
+          expect(Joblin::Batching::Batch).not_to receive(:process_successful_job)
+          expect(Joblin::Batching::Batch).not_to receive(:process_failed_job)
+          expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+
+          BatchTestJobBase.perform_now
+        end
+      end
+
+      context 'when in batch' do
+        let(:bid) { 'SAMPLEBID' }
+
+        context 'when successful' do
+          it 'yields' do
+            expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+            BatchTestJobBase.perform_now
+          end
+
+          it 'calls process_successful_job' do
+            job = BatchTestJobBase.new
+            job.instance_variable_set(:@bid, bid)
+            expect(Joblin::Batching::Batch).to receive(:process_successful_job).with(bid, job.job_id)
+            job.perform_now
+          end
+        end
+
+        context 'when failed' do
+          it 'calls process_failed_job and reraises exception' do
+            reraised = false
+            job = FailingBatchTestJobBase.new
+            job.instance_variable_set(:@bid, bid)
+            expect(Joblin::Batching::Batch).to receive(:process_failed_job)
+            begin
+              job.perform_now
+            rescue
+              reraised = true
+            end
+            expect(reraised).to be_truthy
+          end
+        end
+      end
+    end
+
+    context "When Enqueueing" do
+      context 'when without batch' do
+        it 'just yields' do
+          expect(Joblin::Batching::Batch).not_to receive(:append_jobs)
+          BatchTestJobBase.perform_later
+          expect(BatchTestJobBase).to have_been_enqueued
+        end
+      end
+
+      context 'when in batch' do
+        let(:bid) { 'SAMPLEBID' }
+
+        before do
+          Thread.current[Joblin::Batching::CURRENT_BATCH_THREAD_KEY] = Joblin::Batching::Batch.new(bid)
+          Thread.current[Joblin::Batching::CURRENT_BATCH_THREAD_KEY].instance_variable_set(:@open, true)
+        end
+        after { Thread.current[Joblin::Batching::CURRENT_BATCH_THREAD_KEY] = nil }
+
+        it 'yields' do
+          expect {
+            BatchTestJobBase.perform_later
+          }.to enqueue_job(BatchTestJobBase)
+        end
+
+        it 'assigns bid to job metadata' do
+          job = BatchTestJobBase.perform_later
+          expect(job.bid).to eq bid
+          expect(job.serialize['batch_id']).to eq bid
+        end
+      end
+    end
+
+    context 'worker' do
+      it 'defines method bid' do
+        expect(ActiveJob::Base.instance_methods).to include(:bid)
+      end
+
+      it 'defines method batch' do
+        expect(ActiveJob::Base.instance_methods).to include(:batch)
+      end
+
+      it 'defines method valid_within_batch?' do
+        expect(ActiveJob::Base.instance_methods).to include(:valid_within_batch?)
+      end
+    end
+  end
+
+  describe ".handle_job_death" do
+
+  end
+end