canvas_sync 0.17.1 → 0.17.4

data/spec/job_batching/batch_aware_job_spec.rb

@@ -69,6 +69,7 @@ RSpec.describe CanvasSync::JobBatches::BatchAwareJob do
         Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid)
         Thread.current[:batch].instance_variable_set(:@open, true)
       end
+      after { Thread.current[:batch] = nil }
 
       it 'yields' do
         expect {

data/spec/job_batching/batch_spec.rb

@@ -193,6 +193,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'success' do
       before { batch.on(:complete, Object) }
+
       it 'tries to call complete callback' do
         expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
         expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
@@ -208,6 +209,61 @@ RSpec.describe CanvasSync::JobBatches::Batch do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
       end
 
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        Sidekiq::Testing.inline! do
+          CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
+          CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+        end
+      end
+
+      it 'triggers callbacks as expected' do
+        ActiveJob::Base.queue_adapter = :sidekiq
+        CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 0) }
+
+        class RetryingJob < BatchTestJobBase
+          @@failed = false
+
+          def perform
+            unless @@failed
+              @@failed = true
+              raise "A Failure"
+            end
+          end
+        end
+
+        callback_instance = double('SampleCallback')
+        expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
+        expect(callback_instance).to receive(:on_complete)
+        expect(callback_instance).to receive(:on_success)
+
+        batch.on(:complete, SampleCallback)
+        batch.on(:success, SampleCallback)
+
+        batch.jobs do
+          RetryingJob.perform_later
+        end
+
+        job_def = Sidekiq::Worker.jobs[0]
+        int_job_class = job_def["class"].constantize
+
+        begin
+          int_job_class.process_job(job_def)
+        rescue
+        end
+        Sidekiq::Worker.drain_all
+      end
+
       it 'cleanups redis key' do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
         expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
@@ -216,7 +272,6 @@ RSpec.describe CanvasSync::JobBatches::Batch do
   end
 
   describe '#increment_job_queue' do
-    let(:bid) { 'BID' }
     let(:batch) { CanvasSync::JobBatches::Batch.new }
 
     it 'increments pending' do
@@ -224,12 +279,6 @@ RSpec.describe CanvasSync::JobBatches::Batch do
       pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
       expect(pending).to eq('1')
     end
-
-    it 'increments total' do
-      batch.jobs do TestWorker.perform_async end
-      total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
-      expect(total).to eq('1')
-    end
   end
 
   describe '#enqueue_callbacks' do
@@ -241,6 +290,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
       context 'when no callbacks are defined' do
         it 'clears redis keys' do
           batch = CanvasSync::JobBatches::Batch.new
+          batch.jobs {}
           expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
           CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
         end
@@ -260,8 +310,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'With ActiveJob Adapter' do
       around(:all) do |block|
-        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
-        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker
         block.run
       end
 
@@ -279,12 +328,15 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           let(:opts) { { 'a' => 'b' } }
 
           it 'calls it passing options' do
+            ActiveJob::Base.queue_adapter = :test
+
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
+            batch.jobs {}
 
             CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
 
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback', event.to_s, opts, batch.bid, nil
             )
           end
@@ -295,15 +347,18 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           let(:opts2) { { 'b' => 'a' } }
 
           it 'enqueues each callback passing their options' do
+            ActiveJob::Base.queue_adapter = :test
+
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
             batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
 
             CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback2', event.to_s, opts2, batch.bid, nil
             )
-            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+            expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
               'SampleCallback', event.to_s, opts, batch.bid, nil
             )
           end
@@ -313,8 +368,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
 
     context 'With Sidekiq Adapter' do
       around(:all) do |block|
-        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
-        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
+        CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
         block.run
       end
 
@@ -334,9 +388,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
           it 'calls it passing options' do
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
+            batch.jobs{}
 
             expect(Sidekiq::Client).to receive(:push_bulk).with(
-              'class' => Sidekiq::Batch::Callback::Worker,
+              'class' => Sidekiq::Batch::Callback.worker_class,
               'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
               'queue' => 'default'
             )
@@ -353,9 +408,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
             batch = CanvasSync::JobBatches::Batch.new
             batch.on(event, SampleCallback, opts)
             batch.on(event, SampleCallback2, opts2)
+            batch.jobs{}
 
             expect(Sidekiq::Client).to receive(:push_bulk).with(
-              'class' => Sidekiq::Batch::Callback::Worker,
+              'class' => Sidekiq::Batch::Callback.worker_class,
               'args' => [
                 ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
                 ['SampleCallback', event.to_s, opts, batch.bid, nil]

data/spec/job_batching/callback_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-RSpec.describe CanvasSync::JobBatches::Batch::Callback::Worker do
+RSpec.describe CanvasSync::JobBatches::Batch::Callback.worker_class do
   describe '#perform' do
     it 'does not do anything if it cannot find the callback class' do
       subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')

data/spec/job_batching/context_hash_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::ContextHash do
+
+  class ContextedJob < BatchTestJobBase
+    def perform
+      was_performed(batch_context.flatten)
+    end
+
+    def was_performed(*args); end
+
+    def self.callback_perform(*args)
+      perform_later
+    end
+  end
+
+  it "works with first-level jobs" do
+    ActiveJob::Base.queue_adapter = :sidekiq
+    b = CanvasSync::JobBatches::Batch.new
+    b.context = { hello: 'world' }
+    expect_any_instance_of(ContextedJob).to receive(:was_performed).with({ 'hello' => 'world' })
+    b.jobs do
+      ContextedJob.perform_later
+    end
+    Sidekiq::Worker.drain_all
+  end
+
+  it "works with nested-batch jobs" do
+    ActiveJob::Base.queue_adapter = :sidekiq
+    b = CanvasSync::JobBatches::Batch.new
+    b.context = { hello: 'world', foo: 'bar' }
+    expect_any_instance_of(ContextedJob).to receive(:was_performed).with({ 'hello' => 'world', 'foo' => 'baz', 'some' => 'other' })
+    b.jobs do
+      b2 = CanvasSync::JobBatches::Batch.new
+      b2.context = { some: 'other', foo: 'baz' }
+      b2.jobs do
+        ContextedJob.perform_later
+      end
+    end
+    Sidekiq::Worker.drain_all
+  end
+
+  it 'works with a callback batch' do
+    ActiveJob::Base.queue_adapter = :sidekiq
+    b = CanvasSync::JobBatches::Batch.new
+    b.context = { hello: 'world' }
+    b.on(:success, "ContextedJob.callback_perform")
+    expect_any_instance_of(ContextedJob).to receive(:was_performed).with({ 'hello' => 'world' })
+    b.jobs do
+      BatchTestJobBase.perform_later
+    end
+    Sidekiq::Worker.drain_all
+  end
+end

data/spec/job_batching/flow_spec.rb

@@ -21,30 +21,24 @@ RSpec.describe 'Batch flow' do
     before { batch.on(:complete, SampleCallback, :id => 42) }
     before { batch.description = 'describing the batch' }
     let(:status) { CanvasSync::JobBatches::Batch::Status.new(batch.bid) }
-    let(:jids) { batch.jobs do 3.times do TestWorker.perform_async end end }
     let(:queue) { Sidekiq::Queue.new }
 
     it 'correctly initializes' do
-      expect(jids.size).to eq(3)
-
       expect(batch.bid).not_to be_nil
       expect(batch.description).to eq('describing the batch')
 
-      batch.jobs {}
+      batch.jobs do
+        3.times do
+          TestWorker.perform_async
+        end
+      end
 
-      expect(status.total).to eq(3)
       expect(status.pending).to eq(3)
       expect(status.failures).to eq(0)
       expect(status.complete?).to be false
       expect(status.created_at).not_to be_nil
       expect(status.bid).to eq(batch.bid)
     end
-
-    it 'handles an empty batch' do
-      batch = CanvasSync::JobBatches::Batch.new
-      jids = batch.jobs do nil end
-      expect(jids.size).to eq(0)
-    end
   end
 
   context 'when handling a nested batch' do

data/spec/job_batching/integration/fail_then_succeed.rb

@@ -0,0 +1,42 @@
+require_relative '../integration_helper'
+
+# Workflow when a Job fails, retries, and then succeeds
+
+class Worker1
+  include Sidekiq::Worker
+  sidekiq_options retry: 5
+
+  @@failed = false
+
+  def perform
+    Sidekiq.logger.info "Work 1"
+
+    unless @@failed
+      @@failed = true
+      raise "One Failure"
+    end
+  end
+end
+
+class MyCallback
+  def on_success(status, options)
+    Sidekiq.logger.info "Overall Success #{options} #{status.data}"
+  end
+  alias_method :multi, :on_success
+
+  def on_complete(status, options)
+    Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
+  end
+end
+
+overall = CanvasSync::JobBatches::Batch.new
+overall.on(:success, MyCallback, to: 'success@gmail.com')
+overall.on(:complete, MyCallback, to: 'success@gmail.com')
+overall.jobs do
+  Worker1.perform_async
+end
+
+puts "Overall bid #{overall.bid}"
+
+output, keys = process_tests
+overall_tests(output, keys, file: __FILE__)

data/spec/job_batching/integration_helper.rb

@@ -1,9 +1,8 @@
 require 'spec_helper'
-require 'sidekiq/batch'
 require 'sidekiq/testing'
 
 Sidekiq::Testing.server_middleware do |chain|
-  chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
+  chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
 end
 
 Sidekiq.redis { |r| r.flushdb }
@@ -31,8 +30,11 @@ def process_tests
   [output, keys]
 end
 
-def overall_tests output, keys
-  describe "sidekiq batch" do
+def overall_tests(output, keys, file: nil)
+  test_name = "Batch Integration Test"
+  test_name = File.basename(file, ".*")  if file
+
+  Rspec.describe test_name do
     it "runs overall complete callback" do
       expect(output).to include "Overall Complete"
     end

data/spec/job_batching/sidekiq_spec.rb

@@ -58,6 +58,7 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
       let(:bid) { 'SAMPLEBID' }
       let(:jid) { 'SAMPLEJID' }
       before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
+      after { Thread.current[:batch] = nil }
 
       it 'yields' do
         yielded = false

data/spec/job_batching/status_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 
 RSpec.describe CanvasSync::JobBatches::Batch::Status do
-  let(:bid) { 'BID' }
-  let(:batch) { CanvasSync::JobBatches::Batch.new(bid) }
+  let(:batch) { CanvasSync::JobBatches::Batch.new() }
+  let(:bid) { batch.bid }
   subject { described_class.new(bid) }
 
   describe '#join' do
@@ -51,7 +51,7 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
     end
 
     context 'when with error' do
-      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'jid123') }
+      before { batch.jobs{}; CanvasSync::JobBatches::Batch.process_failed_job(bid, 'jid123') }
 
       it 'returns array with failed jids' do
         expect(subject.failure_info).to eq(['jid123'])
@@ -59,25 +59,9 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
     end
   end
 
-  describe '#total' do
-    context 'when not initalized' do
-      it 'returns 0 failed jobs' do
-        expect(subject.total).to eq(0)
-      end
-    end
-
-    context 'when more than 0' do
-      before { batch.jobs do TestWorker.perform_async end }
-
-      it 'returns failed jobs' do
-        expect(subject.total).to eq(1)
-      end
-    end
-  end
-
   describe '#data' do
     it 'returns batch description' do
-      expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+      expect(subject.data).to include(failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
     end
   end
 

data/spec/spec_helper.rb

@@ -15,17 +15,13 @@ require 'pry-nav'
 
 require 'sidekiq/testing'
 Sidekiq::Testing.fake!
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
+end
 
 require 'fakeredis/rspec'
 Dir[File.dirname(__FILE__) + "/job_batching/support/**/*.rb"].each {|f| require f }
 
-# Fix an issue with fakeredis with Redis >=4.2
-class Redis::Connection::Memory
-  def exists(*keys)
-    keys.count { |key| data.key?(key) }
-  end
-end
-
 ActiveRecord::Migration.maintain_test_schema!
 
 RSpec.configure do |config|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.17.4
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-09 00:00:00.000000000 Z
+date: 2020-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,20 +234,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: fakeredis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -427,10 +413,23 @@ files:
 - lib/canvas_sync/job_batches/callback.rb
 - lib/canvas_sync/job_batches/chain_builder.rb
 - lib/canvas_sync/job_batches/context_hash.rb
+- lib/canvas_sync/job_batches/hincr_max.lua
 - lib/canvas_sync/job_batches/jobs/base_job.rb
 - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
+- lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
 - lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
+- lib/canvas_sync/job_batches/pool.rb
+- lib/canvas_sync/job_batches/redis_model.rb
+- lib/canvas_sync/job_batches/redis_script.rb
 - lib/canvas_sync/job_batches/sidekiq.rb
+- lib/canvas_sync/job_batches/sidekiq/web.rb
+- lib/canvas_sync/job_batches/sidekiq/web/helpers.rb
+- lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb
+- lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb
+- lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb
+- lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb
+- lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb
+- lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb
 - lib/canvas_sync/job_batches/status.rb
 - lib/canvas_sync/jobs/begin_sync_chain_job.rb
 - lib/canvas_sync/jobs/report_checker.rb
@@ -597,7 +596,9 @@ files:
 - spec/job_batching/batch_aware_job_spec.rb
 - spec/job_batching/batch_spec.rb
 - spec/job_batching/callback_spec.rb
+- spec/job_batching/context_hash_spec.rb
 - spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/fail_then_succeed.rb
 - spec/job_batching/integration/integration.rb
 - spec/job_batching/integration/nested.rb
 - spec/job_batching/integration/simple.rb
@@ -789,7 +790,9 @@ test_files:
 - spec/job_batching/batch_aware_job_spec.rb
 - spec/job_batching/batch_spec.rb
 - spec/job_batching/callback_spec.rb
+- spec/job_batching/context_hash_spec.rb
 - spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/fail_then_succeed.rb
 - spec/job_batching/integration/integration.rb
 - spec/job_batching/integration/nested.rb
 - spec/job_batching/integration/simple.rb