canvas_sync 0.17.0 → 0.17.3.beta3

data/spec/job_batching/batch_aware_job_spec.rb

@@ -69,6 +69,7 @@ RSpec.describe CanvasSync::JobBatches::BatchAwareJob do
         Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid)
         Thread.current[:batch].instance_variable_set(:@open, true)
       end
+      after { Thread.current[:batch] = nil }
 
       it 'yields' do
         expect {

data/spec/job_batching/batch_spec.rb

@@ -193,6 +193,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'success' do
       before { batch.on(:complete, Object) }
+
       it 'tries to call complete callback' do
         expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
         expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
@@ -208,6 +209,61 @@ RSpec.describe CanvasSync::JobBatches::Batch do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
       end
 
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        Sidekiq::Testing.inline! do
+          CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
+          CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+        end
+      end
+
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 0) }
+
+        class RetryingJob < BatchTestJobBase
+          @@failed = false
+
+          def perform
+            unless @@failed
+              @@failed = true
+              raise "A Failure"
+            end
+          end
+        end
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        batch.jobs do
+          RetryingJob.perform_later
+        end
+
+        job_def = Sidekiq::Worker.jobs[0]
+        int_job_class = job_def["class"].constantize
+
+        begin
+          int_job_class.process_job(job_def)
+        rescue
+        end
+        Sidekiq::Worker.drain_all
+      end
+
       it 'cleanups redis key' do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
         expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
@@ -224,12 +280,6 @@ RSpec.describe CanvasSync::JobBatches::Batch do
       pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
       expect(pending).to eq('1')
     end
-
-    it 'increments total' do
-      batch.jobs do TestWorker.perform_async end
-      total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
-      expect(total).to eq('1')
-    end
   end
 
   describe '#enqueue_callbacks' do
@@ -241,6 +291,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
       context 'when no callbacks are defined' do
         it 'clears redis keys' do
           batch = CanvasSync::JobBatches::Batch.new
+          batch.jobs {}
           expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
           CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
         end
@@ -260,8 +311,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'With ActiveJob Adapter' do
       around(:all) do |block|
-        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
-        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker
         block.run
       end
 
@@ -279,12 +329,15 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           let(:opts) { { 'a' => 'b' } }
 
           it 'calls it passing options' do
+            ActiveJob::Base.queue_adapter = :test
+
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
+            batch.jobs {}
 
             CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
 
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback', event.to_s, opts, batch.bid, nil
             )
           end
@@ -295,15 +348,18 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           let(:opts2) { { 'b' => 'a' } }
 
           it 'enqueues each callback passing their options' do
+            ActiveJob::Base.queue_adapter = :test
+
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
             batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
 
             CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback2', event.to_s, opts2, batch.bid, nil
             )
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback', event.to_s, opts, batch.bid, nil
             )
           end
@@ -313,8 +369,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'With Sidekiq Adapter' do
       around(:all) do |block|
-        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
-        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
         block.run
       end
 
@@ -334,9 +389,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           it 'calls it passing options' do
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
+            batch.jobs{}
 
             expect(Sidekiq::Client).to receive(:push_bulk).with(
-              'class' => Sidekiq::Batch::Callback::Worker,
+              'class' => Sidekiq::Batch::Callback.worker_class,
               'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
               'queue' => 'default'
             )
@@ -353,9 +409,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
             batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
 
             expect(Sidekiq::Client).to receive(:push_bulk).with(
-              'class' => Sidekiq::Batch::Callback::Worker,
+              'class' => Sidekiq::Batch::Callback.worker_class,
               'args' => [
                 ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
                 ['SampleCallback', event.to_s, opts, batch.bid, nil]

data/spec/job_batching/callback_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe CanvasSync::JobBatches::Batch::Callback::Worker do
+RSpec.describe CanvasSync::JobBatches::Batch::Callback.worker_class do
   describe '#perform' do
     it 'does not do anything if it cannot find the callback class' do
       subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')

data/spec/job_batching/flow_spec.rb

@@ -21,30 +21,24 @@ RSpec.describe 'Batch flow' do
     before { batch.on(:complete, SampleCallback, :id => 42) }
     before { batch.description = 'describing the batch' }
     let(:status) { CanvasSync::JobBatches::Batch::Status.new(batch.bid) }
-    let(:jids) { batch.jobs do 3.times do TestWorker.perform_async end end }
     let(:queue) { Sidekiq::Queue.new }
 
     it 'correctly initializes' do
-      expect(jids.size).to eq(3)
-
       expect(batch.bid).not_to be_nil
       expect(batch.description).to eq('describing the batch')
 
-      batch.jobs {}
+      batch.jobs do
+        3.times do
+          TestWorker.perform_async
+        end
+      end
 
-      expect(status.total).to eq(3)
       expect(status.pending).to eq(3)
       expect(status.failures).to eq(0)
       expect(status.complete?).to be false
       expect(status.created_at).not_to be_nil
       expect(status.bid).to eq(batch.bid)
     end
-
-    it 'handles an empty batch' do
-      batch = CanvasSync::JobBatches::Batch.new
-      jids = batch.jobs do nil end
-      expect(jids.size).to eq(0)
-    end
   end
 
   context 'when handling a nested batch' do

data/spec/job_batching/integration/fail_then_succeed.rb

@@ -0,0 +1,42 @@
+require_relative '../integration_helper'
+
+# Workflow when a Job fails, retries, and then succeeds
+
+class Worker1
+  include Sidekiq::Worker
+  sidekiq_options retry: 5
+
+  @@failed = false
+
+  def perform
+    Sidekiq.logger.info "Work 1"
+
+    unless @@failed
+      @@failed = true
+      raise "One Failure"
+    end
+  end
+end
+
+class MyCallback
+  def on_success(status, options)
+    Sidekiq.logger.info "Overall Success #{options} #{status.data}"
+  end
+  alias_method :multi, :on_success
+
+  def on_complete(status, options)
+    Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
+  end
+end
+
+overall = CanvasSync::JobBatches::Batch.new
+overall.on(:success, MyCallback, to: 'success@gmail.com')
+overall.on(:complete, MyCallback, to: 'success@gmail.com')
+overall.jobs do
+  Worker1.perform_async
+end
+
+puts "Overall bid #{overall.bid}"
+
+output, keys = process_tests
+overall_tests(output, keys, file: __FILE__)

data/spec/job_batching/integration_helper.rb

@@ -1,9 +1,8 @@
 require 'spec_helper'
-require 'sidekiq/batch'
 require 'sidekiq/testing'
 
 Sidekiq::Testing.server_middleware do |chain|
-  chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
+  chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
 end
 
 Sidekiq.redis { |r| r.flushdb }
@@ -31,8 +30,11 @@ def process_tests
   [output, keys]
 end
 
-def overall_tests output, keys
-  describe "sidekiq batch" do
+def overall_tests(output, keys, file: nil)
+  test_name = "Batch Integration Test"
+  test_name = File.basename(file, ".*")  if file
+
+  Rspec.describe test_name do
     it "runs overall complete callback" do
       expect(output).to include "Overall Complete"
     end

data/spec/job_batching/sidekiq_spec.rb

@@ -58,6 +58,7 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
       let(:bid) { 'SAMPLEBID' }
       let(:jid) { 'SAMPLEJID' }
       before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
+      after { Thread.current[:batch] = nil }
 
       it 'yields' do
         yielded = false

data/spec/job_batching/status_spec.rb

@@ -59,25 +59,9 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
     end
   end
 
-  describe '#total' do
-    context 'when not initalized' do
-      it 'returns 0 failed jobs' do
-        expect(subject.total).to eq(0)
-      end
-    end
-
-    context 'when more than 0' do
-      before { batch.jobs do TestWorker.perform_async end }
-
-      it 'returns failed jobs' do
-        expect(subject.total).to eq(1)
-      end
-    end
-  end
-
   describe '#data' do
     it 'returns batch description' do
-      expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+      expect(subject.data).to include(failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.17.3.beta3
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-05 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -427,9 +427,14 @@ files:
 - lib/canvas_sync/job_batches/callback.rb
 - lib/canvas_sync/job_batches/chain_builder.rb
 - lib/canvas_sync/job_batches/context_hash.rb
+- lib/canvas_sync/job_batches/hincr_max.lua
 - lib/canvas_sync/job_batches/jobs/base_job.rb
 - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
+- lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
 - lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
+- lib/canvas_sync/job_batches/pool.rb
+- lib/canvas_sync/job_batches/redis_model.rb
+- lib/canvas_sync/job_batches/redis_script.rb
 - lib/canvas_sync/job_batches/sidekiq.rb
 - lib/canvas_sync/job_batches/status.rb
 - lib/canvas_sync/jobs/begin_sync_chain_job.rb
@@ -598,6 +603,7 @@ files:
 - spec/job_batching/batch_spec.rb
 - spec/job_batching/callback_spec.rb
 - spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/fail_then_succeed.rb
 - spec/job_batching/integration/integration.rb
 - spec/job_batching/integration/nested.rb
 - spec/job_batching/integration/simple.rb
@@ -643,9 +649,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: 
@@ -790,6 +796,7 @@ test_files:
 - spec/job_batching/batch_spec.rb
 - spec/job_batching/callback_spec.rb
 - spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/fail_then_succeed.rb
 - spec/job_batching/integration/integration.rb
 - spec/job_batching/integration/nested.rb
 - spec/job_batching/integration/simple.rb