canvas_sync 0.16.5 → 0.17.0.beta1

data/spec/job_batching/integration/workflow.rb

@@ -0,0 +1,134 @@
+require 'integration_helper'
+
+# Complex workflow with sequential and nested
+# Also test sub batches without callbacks
+# Batches:
+# - Overall
+#  - Worker1
+#   - Worker3
+#  - Worker2 + Worker3
+#   - Worker1
+#    - Worker3
+#  - Worker4
+#  - Worker5
+
+class Callbacks
+  def worker1 status, opts
+    Sidekiq.logger.info "Success 1 #{status.data}"
+
+    overall = CanvasSync::JobBatches::Batch.new status.parent_bid
+    overall.jobs do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.on(:success, "Callbacks#worker2")
+      batch.jobs do
+        Worker2.perform_async
+      end
+    end
+  end
+
+  def worker2 status, opts
+    Sidekiq.logger.info "Success 2 #{status.data}"
+    overall = CanvasSync::JobBatches::Batch.new status.parent_bid
+    overall.jobs do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.on(:success, "Callbacks#worker4")
+      batch.jobs do
+        Worker4.perform_async
+      end
+    end
+
+  end
+
+  def worker4 status, opts
+    Sidekiq.logger.info "Success 4 #{status.data}"
+    overall = CanvasSync::JobBatches::Batch.new status.parent_bid
+    overall.jobs do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.on(:success, "Callbacks#worker5")
+      batch.jobs do
+        Worker5.perform_async
+      end
+    end
+  end
+
+  def worker5 status, opts
+    Sidekiq.logger.info "Success 5 #{status.data}"
+  end
+end
+
+class Worker1
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work 1"
+    batch = CanvasSync::JobBatches::Batch.new
+    batch.jobs do
+      Worker3.perform_async
+    end
+  end
+end
+
+class Worker2
+  include Sidekiq::Worker
+
+  def perform
+    Sidekiq.logger.info "Work 2"
+    if bid
+      batch.jobs do
+        Worker3.perform_async
+      end
+      newb = CanvasSync::JobBatches::Batch.new
+      newb.jobs do
+        Worker1.perform_async
+      end
+    end
+  end
+end
+
+class Worker3
+  include Sidekiq::Worker
+  def perform
+    Sidekiq.logger.info "Work 3"
+  end
+end
+
+class Worker4
+  include Sidekiq::Worker
+  def perform
+    Sidekiq.logger.info "Work 4"
+  end
+end
+
+class Worker5
+  include Sidekiq::Worker
+  def perform
+    Sidekiq.logger.info "Work 5"
+  end
+end
+
+class MyCallback
+  def on_success(status, options)
+    Sidekiq.logger.info "Overall Success #{options} #{status.data}"
+  end
+  alias_method :multi, :on_success
+
+  def on_complete(status, options)
+    Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
+  end
+end
+
+overall = CanvasSync::JobBatches::Batch.new
+overall.on(:success, MyCallback, to: 'success@gmail.com')
+overall.on(:complete, MyCallback, to: 'success@gmail.com')
+overall.jobs do
+  batch1 = CanvasSync::JobBatches::Batch.new
+  batch1.on(:success, "Callbacks#worker1")
+  batch1.jobs do
+    Worker1.perform_async
+  end
+end
+
+puts "Overall bid #{overall.bid}"
+
+output, keys = process_tests
+overall_tests output, keys

data/spec/job_batching/integration_helper.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'sidekiq/batch'
+require 'sidekiq/testing'
+
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
+end
+
+Sidekiq.redis { |r| r.flushdb }
+
+def redis_keys
+  Sidekiq.redis { |r| r.keys('BID-*') }
+end
+
+def dump_redis_keys
+  puts redis_keys.inspect
+end
+
+def process_tests
+  out_buf = StringIO.new
+  Sidekiq.logger = Logger.new out_buf
+
+  # Sidekiq.logger.level = :info
+
+  Sidekiq::Worker.drain_all
+
+  output = out_buf.string
+  keys = redis_keys
+  puts out_buf.string
+
+  [output, keys]
+end
+
+def overall_tests output, keys
+  describe "sidekiq batch" do
+    it "runs overall complete callback" do
+      expect(output).to include "Overall Complete"
+    end
+
+    it "runs overall success callback" do
+      expect(output).to include "Overall Success"
+    end
+
+    it "cleans redis keys" do
+      expect(keys).to eq([])
+    end
+  end
+end

data/spec/job_batching/sidekiq_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Sidekiq do
+  describe CanvasSync::JobBatches::Sidekiq::ServerMiddleware do
+    context 'when without batch' do
+      it 'just yields' do
+        yielded = false
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_successful_job)
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_failed_job)
+        subject.call(nil, {}, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+      let(:jid) { 'SAMPLEJID' }
+
+      context 'when successful' do
+        it 'yields' do
+          yielded = false
+          subject.call(nil, { 'bid' => bid, 'jid' => jid }, nil) { yielded = true }
+          expect(yielded).to be_truthy
+        end
+
+        it 'calls process_successful_job' do
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(bid, nil)
+          subject.call(nil, { 'bid' => bid }, nil) {}
+        end
+      end
+
+      context 'when failed' do
+        it 'calls process_failed_job and reraises exception' do
+          reraised = false
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_failed_job)
+          begin
+            subject.call(nil, { 'bid' => bid }, nil) { raise 'ERR' }
+          rescue
+            reraised = true
+          end
+          expect(reraised).to be_truthy
+        end
+      end
+    end
+  end
+
+  describe CanvasSync::JobBatches::Sidekiq::ClientMiddleware do
+    context 'when without batch' do
+      it 'just yields' do
+        yielded = false
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:increment_job_queue)
+        subject.call(nil, {}, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+      let(:jid) { 'SAMPLEJID' }
+      before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
+
+      it 'yields' do
+        yielded = false
+        subject.call(nil, { 'jid' => jid }, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+
+      it 'increments job queue' do
+        # expect(CanvasSync::JobBatches::Batch).to receive(:increment_job_queue).with(bid)
+        # subject.call(nil, { 'jid' => jid }, nil) {}
+      end
+
+      it 'assigns bid to msg' do
+        msg = { 'jid' => jid }
+        subject.call(nil, msg, nil) {}
+        expect(msg[:bid]).to eq(bid)
+      end
+    end
+  end
+end
+
+RSpec.describe CanvasSync::JobBatches::Sidekiq do
+  let(:config) { class_double(Sidekiq) }
+  let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
+
+  context 'client' do
+    it 'adds client middleware' do
+      expect(Sidekiq).to receive(:configure_client).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
+      CanvasSync::JobBatches::Sidekiq.configure
+    end
+  end
+
+  context 'server' do
+    let(:server_middleware) { double(Sidekiq::Middleware::Chain) }
+    let(:death_handlers) { double(Array) }
+
+    it 'adds client and server middleware' do
+      expect(Sidekiq).to receive(:configure_server).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(config).to receive(:server_middleware).and_yield(server_middleware)
+      expect(config).to receive(:death_handlers).and_return(death_handlers)
+      expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
+      expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
+      expect(death_handlers).to receive(:<<)
+      CanvasSync::JobBatches::Sidekiq.configure
+    end
+  end
+
+  context 'worker' do
+    it 'defines method bid' do
+      expect(Sidekiq::Worker.instance_methods).to include(:bid)
+    end
+
+    it 'defines method batch' do
+      expect(Sidekiq::Worker.instance_methods).to include(:batch)
+    end
+
+    it 'defines method valid_within_batch?' do
+      expect(Sidekiq::Worker.instance_methods).to include(:valid_within_batch?)
+    end
+  end
+end

data/spec/job_batching/status_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Batch::Status do
+  let(:bid) { 'BID' }
+  let(:batch) { CanvasSync::JobBatches::Batch.new(bid) }
+  subject { described_class.new(bid) }
+
+  describe '#join' do
+    it 'raises info' do
+      expect { subject.join }.to raise_error('Not supported')
+    end
+  end
+
+  describe '#pending' do
+    context 'when not initalized' do
+      it 'returns 0 pending jobs' do
+        expect(subject.pending).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+      it 'returns pending jobs' do
+        expect(subject.pending).to eq(1)
+      end
+    end
+  end
+
+  describe '#failures' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.failures).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.increment_job_queue(bid) }
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'FAILEDID') }
+
+      it 'returns failed jobs' do
+        expect(subject.failures).to eq(1)
+      end
+    end
+  end
+
+  describe '#failure_info' do
+    context 'when not initalized' do
+      it 'returns empty array' do
+        expect(subject.failure_info).to eq([])
+      end
+    end
+
+    context 'when with error' do
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'jid123') }
+
+      it 'returns array with failed jids' do
+        expect(subject.failure_info).to eq(['jid123'])
+      end
+    end
+  end
+
+  describe '#total' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.total).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+
+      it 'returns failed jobs' do
+        expect(subject.total).to eq(1)
+      end
+    end
+  end
+
+  describe '#data' do
+    it 'returns batch description' do
+      expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+    end
+  end
+
+  describe '#created_at' do
+    it 'returns time' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do TestWorker.perform_async end
+      status = described_class.new(batch.bid)
+      expect(status.created_at).not_to be_nil
+    end
+  end
+end

data/spec/job_batching/support/base_job.rb

@@ -0,0 +1,14 @@
+class BatchTestJobBase < ActiveJob::Base
+  def perform
+  end
+
+  def self.perform_async(*args)
+    perform_later(*args)
+  end
+end
+
+class FailingBatchTestJobBase < BatchTestJobBase
+  def perform
+    raise "Foo"
+  end
+end

data/spec/job_batching/support/sample_callback.rb

@@ -0,0 +1,2 @@
+  class SampleCallback; end
+  class SampleCallback2; end

data/spec/spec_helper.rb

@@ -13,6 +13,12 @@ require 'shoulda/matchers'
 require 'pry'
 require 'pry-nav'
 
+require 'sidekiq/testing'
+Sidekiq::Testing.fake!
+
+require 'fakeredis/rspec'
+Dir[File.dirname(__FILE__) + "/job_batching/support/**/*.rb"].each {|f| require f }
+
 ActiveRecord::Migration.maintain_test_schema!
 
 RSpec.configure do |config|
@@ -51,3 +57,7 @@ end
 def canvas_sync_client
   Bearcat::Client.new(token: 'cool-token', prefix: 'http://test.instructure.com')
 end
+
+def set_batch_context(ctx)
+  allow_any_instance_of(ActiveJob::Base).to receive(:batch_context).and_return(ctx.with_indifferent_access)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.16.5
+  version: 0.17.0.beta1
 platform: ruby
 authors:
 - Nate Collings
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-27 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,6 +234,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
@@ -248,6 +276,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -304,7 +346,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+description: 
 email:
 - ncollings@instructure.com
 executables: []
@@ -316,10 +358,12 @@ files:
 - app/controllers/api/v1/health_check_controller.rb
 - app/controllers/api/v1/live_events_controller.rb
 - app/models/canvas_sync/job_log.rb
+- app/models/canvas_sync/sync_batch.rb
 - config/initializers/apartment.rb
 - db/migrate/20170915210836_create_canvas_sync_job_log.rb
 - db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb
 - db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb
+- db/migrate/20201018210836_create_canvas_sync_sync_batches.rb
 - lib/canvas_sync.rb
 - lib/canvas_sync/api_syncable.rb
 - lib/canvas_sync/class_callback_executor.rb
@@ -377,8 +421,17 @@ files:
 - lib/canvas_sync/importers/bulk_importer.rb
 - lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
-- lib/canvas_sync/job_chain.rb
-- lib/canvas_sync/jobs/fork_gather.rb
+- lib/canvas_sync/job_batches/batch.rb
+- lib/canvas_sync/job_batches/batch_aware_job.rb
+- lib/canvas_sync/job_batches/callback.rb
+- lib/canvas_sync/job_batches/chain_builder.rb
+- lib/canvas_sync/job_batches/context_hash.rb
+- lib/canvas_sync/job_batches/jobs/base_job.rb
+- lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
+- lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
+- lib/canvas_sync/job_batches/sidekiq.rb
+- lib/canvas_sync/job_batches/status.rb
+- lib/canvas_sync/jobs/begin_sync_chain_job.rb
 - lib/canvas_sync/jobs/report_checker.rb
 - lib/canvas_sync/jobs/report_processor_job.rb
 - lib/canvas_sync/jobs/report_starter.rb
@@ -407,7 +460,6 @@ files:
 - lib/canvas_sync/sidekiq_job.rb
 - lib/canvas_sync/version.rb
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -524,9 +576,6 @@ files:
 - spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
 - spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb
 - spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/development.log
-- spec/dummy/log/test.log
 - spec/factories/account_factory.rb
 - spec/factories/admin_factory.rb
 - spec/factories/assignment_factory.rb
@@ -542,6 +591,19 @@ files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json
@@ -558,8 +620,6 @@ files:
 - spec/support/fixtures/reports/group_memberships.csv
 - spec/support/fixtures/reports/groups.csv
 - spec/support/fixtures/reports/provisioning_csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
 - spec/support/fixtures/reports/sections.csv
 - spec/support/fixtures/reports/submissions.csv
 - spec/support/fixtures/reports/users.csv
@@ -567,7 +627,7 @@ files:
 homepage: https://instructure.com
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -578,17 +638,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
 specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -705,9 +765,6 @@ test_files:
 - spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
 - spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb
 - spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/development.log
-- spec/dummy/log/test.log
 - spec/factories/account_factory.rb
 - spec/factories/admin_factory.rb
 - spec/factories/assignment_factory.rb
@@ -723,6 +780,19 @@ test_files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json
@@ -739,8 +809,6 @@ test_files:
 - spec/support/fixtures/reports/group_memberships.csv
 - spec/support/fixtures/reports/groups.csv
 - spec/support/fixtures/reports/provisioning_csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv
-- spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv
 - spec/support/fixtures/reports/sections.csv
 - spec/support/fixtures/reports/submissions.csv
 - spec/support/fixtures/reports/users.csv