canvas_sync 0.16.2 → 0.17.0.beta3

data/spec/job_batching/integration_helper.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+require 'sidekiq/batch'
+require 'sidekiq/testing'
+
+Sidekiq::Testing.server_middleware do |chain|
+  chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
+end
+
+Sidekiq.redis { |r| r.flushdb }
+
+def redis_keys
+  Sidekiq.redis { |r| r.keys('BID-*') }
+end
+
+def dump_redis_keys
+  puts redis_keys.inspect
+end
+
+def process_tests
+  out_buf = StringIO.new
+  Sidekiq.logger = Logger.new out_buf
+
+  # Sidekiq.logger.level = :info
+
+  Sidekiq::Worker.drain_all
+
+  output = out_buf.string
+  keys = redis_keys
+  puts out_buf.string
+
+  [output, keys]
+end
+
+def overall_tests output, keys
+  describe "sidekiq batch" do
+    it "runs overall complete callback" do
+      expect(output).to include "Overall Complete"
+    end
+
+    it "runs overall success callback" do
+      expect(output).to include "Overall Success"
+    end
+
+    it "cleans redis keys" do
+      expect(keys).to eq([])
+    end
+  end
+end

data/spec/job_batching/sidekiq_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Sidekiq do
+  describe CanvasSync::JobBatches::Sidekiq::ServerMiddleware do
+    context 'when without batch' do
+      it 'just yields' do
+        yielded = false
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_successful_job)
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_failed_job)
+        subject.call(nil, {}, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+      let(:jid) { 'SAMPLEJID' }
+
+      context 'when successful' do
+        it 'yields' do
+          yielded = false
+          subject.call(nil, { 'bid' => bid, 'jid' => jid }, nil) { yielded = true }
+          expect(yielded).to be_truthy
+        end
+
+        it 'calls process_successful_job' do
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(bid, nil)
+          subject.call(nil, { 'bid' => bid }, nil) {}
+        end
+      end
+
+      context 'when failed' do
+        it 'calls process_failed_job and reraises exception' do
+          reraised = false
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_failed_job)
+          begin
+            subject.call(nil, { 'bid' => bid }, nil) { raise 'ERR' }
+          rescue
+            reraised = true
+          end
+          expect(reraised).to be_truthy
+        end
+      end
+    end
+  end
+
+  describe CanvasSync::JobBatches::Sidekiq::ClientMiddleware do
+    context 'when without batch' do
+      it 'just yields' do
+        yielded = false
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:increment_job_queue)
+        subject.call(nil, {}, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+      let(:jid) { 'SAMPLEJID' }
+      before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
+
+      it 'yields' do
+        yielded = false
+        subject.call(nil, { 'jid' => jid }, nil) { yielded = true }
+        expect(yielded).to be_truthy
+      end
+
+      it 'increments job queue' do
+        # expect(CanvasSync::JobBatches::Batch).to receive(:increment_job_queue).with(bid)
+        # subject.call(nil, { 'jid' => jid }, nil) {}
+      end
+
+      it 'assigns bid to msg' do
+        msg = { 'jid' => jid }
+        subject.call(nil, msg, nil) {}
+        expect(msg[:bid]).to eq(bid)
+      end
+    end
+  end
+end
+
+RSpec.describe CanvasSync::JobBatches::Sidekiq do
+  let(:config) { class_double(Sidekiq) }
+  let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
+
+  context 'client' do
+    it 'adds client middleware' do
+      expect(Sidekiq).to receive(:configure_client).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
+      CanvasSync::JobBatches::Sidekiq.configure
+    end
+  end
+
+  context 'server' do
+    let(:server_middleware) { double(Sidekiq::Middleware::Chain) }
+    let(:death_handlers) { double(Array) }
+
+    it 'adds client and server middleware' do
+      expect(Sidekiq).to receive(:configure_server).and_yield(config)
+      expect(config).to receive(:client_middleware).and_yield(client_middleware)
+      expect(config).to receive(:server_middleware).and_yield(server_middleware)
+      expect(config).to receive(:death_handlers).and_return(death_handlers)
+      expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
+      expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
+      expect(death_handlers).to receive(:<<)
+      CanvasSync::JobBatches::Sidekiq.configure
+    end
+  end
+
+  context 'worker' do
+    it 'defines method bid' do
+      expect(Sidekiq::Worker.instance_methods).to include(:bid)
+    end
+
+    it 'defines method batch' do
+      expect(Sidekiq::Worker.instance_methods).to include(:batch)
+    end
+
+    it 'defines method valid_within_batch?' do
+      expect(Sidekiq::Worker.instance_methods).to include(:valid_within_batch?)
+    end
+  end
+end

data/spec/job_batching/status_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::Batch::Status do
+  let(:bid) { 'BID' }
+  let(:batch) { CanvasSync::JobBatches::Batch.new(bid) }
+  subject { described_class.new(bid) }
+
+  describe '#join' do
+    it 'raises info' do
+      expect { subject.join }.to raise_error('Not supported')
+    end
+  end
+
+  describe '#pending' do
+    context 'when not initalized' do
+      it 'returns 0 pending jobs' do
+        expect(subject.pending).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+      it 'returns pending jobs' do
+        expect(subject.pending).to eq(1)
+      end
+    end
+  end
+
+  describe '#failures' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.failures).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.increment_job_queue(bid) }
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'FAILEDID') }
+
+      it 'returns failed jobs' do
+        expect(subject.failures).to eq(1)
+      end
+    end
+  end
+
+  describe '#failure_info' do
+    context 'when not initalized' do
+      it 'returns empty array' do
+        expect(subject.failure_info).to eq([])
+      end
+    end
+
+    context 'when with error' do
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'jid123') }
+
+      it 'returns array with failed jids' do
+        expect(subject.failure_info).to eq(['jid123'])
+      end
+    end
+  end
+
+  describe '#total' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.total).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do TestWorker.perform_async end }
+
+      it 'returns failed jobs' do
+        expect(subject.total).to eq(1)
+      end
+    end
+  end
+
+  describe '#data' do
+    it 'returns batch description' do
+      expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+    end
+  end
+
+  describe '#created_at' do
+    it 'returns time' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do TestWorker.perform_async end
+      status = described_class.new(batch.bid)
+      expect(status.created_at).not_to be_nil
+    end
+  end
+end

data/spec/job_batching/support/base_job.rb

@@ -0,0 +1,14 @@
+class BatchTestJobBase < ActiveJob::Base
+  def perform
+  end
+
+  def self.perform_async(*args)
+    perform_later(*args)
+  end
+end
+
+class FailingBatchTestJobBase < BatchTestJobBase
+  def perform
+    raise "Foo"
+  end
+end

data/spec/job_batching/support/sample_callback.rb

@@ -0,0 +1,2 @@
+  class SampleCallback; end
+  class SampleCallback2; end

data/spec/spec_helper.rb

@@ -13,6 +13,19 @@ require 'shoulda/matchers'
 require 'pry'
 require 'pry-nav'
 
+require 'sidekiq/testing'
+Sidekiq::Testing.fake!
+
+require 'fakeredis/rspec'
+Dir[File.dirname(__FILE__) + "/job_batching/support/**/*.rb"].each {|f| require f }
+
+# Fix an issue with fakeredis with Redis >=4.2
+class Redis::Connection::Memory
+  def exists(*keys)
+    keys.count { |key| data.key?(key) }
+  end
+end
+
 ActiveRecord::Migration.maintain_test_schema!
 
 RSpec.configure do |config|
@@ -51,3 +64,7 @@ end
 def canvas_sync_client
   Bearcat::Client.new(token: 'cool-token', prefix: 'http://test.instructure.com')
 end
+
+def set_batch_context(ctx)
+  allow_any_instance_of(ActiveJob::Base).to receive(:batch_context).and_return(ctx.with_indifferent_access)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.17.0.beta3
 platform: ruby
 authors:
 - Nate Collings
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2020-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,6 +234,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
@@ -248,6 +276,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -316,10 +358,12 @@ files:
 - app/controllers/api/v1/health_check_controller.rb
 - app/controllers/api/v1/live_events_controller.rb
 - app/models/canvas_sync/job_log.rb
+- app/models/canvas_sync/sync_batch.rb
 - config/initializers/apartment.rb
 - db/migrate/20170915210836_create_canvas_sync_job_log.rb
 - db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb
 - db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb
+- db/migrate/20201018210836_create_canvas_sync_sync_batches.rb
 - lib/canvas_sync.rb
 - lib/canvas_sync/api_syncable.rb
 - lib/canvas_sync/class_callback_executor.rb
@@ -377,8 +421,17 @@ files:
 - lib/canvas_sync/importers/bulk_importer.rb
 - lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
-- lib/canvas_sync/job_chain.rb
-- lib/canvas_sync/jobs/fork_gather.rb
+- lib/canvas_sync/job_batches/batch.rb
+- lib/canvas_sync/job_batches/batch_aware_job.rb
+- lib/canvas_sync/job_batches/callback.rb
+- lib/canvas_sync/job_batches/chain_builder.rb
+- lib/canvas_sync/job_batches/context_hash.rb
+- lib/canvas_sync/job_batches/jobs/base_job.rb
+- lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
+- lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
+- lib/canvas_sync/job_batches/sidekiq.rb
+- lib/canvas_sync/job_batches/status.rb
+- lib/canvas_sync/jobs/begin_sync_chain_job.rb
 - lib/canvas_sync/jobs/report_checker.rb
 - lib/canvas_sync/jobs/report_processor_job.rb
 - lib/canvas_sync/jobs/report_starter.rb
@@ -393,6 +446,7 @@ files:
 - lib/canvas_sync/jobs/sync_simple_table_job.rb
 - lib/canvas_sync/jobs/sync_submissions_job.rb
 - lib/canvas_sync/jobs/sync_terms_job.rb
+- lib/canvas_sync/misc_helper.rb
 - lib/canvas_sync/processors/assignment_groups_processor.rb
 - lib/canvas_sync/processors/assignments_processor.rb
 - lib/canvas_sync/processors/context_module_items_processor.rb
@@ -406,7 +460,6 @@ files:
 - lib/canvas_sync/sidekiq_job.rb
 - lib/canvas_sync/version.rb
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -458,6 +511,7 @@ files:
 - spec/dummy/app/models/enrollment.rb
 - spec/dummy/app/models/group.rb
 - spec/dummy/app/models/group_membership.rb
+- spec/dummy/app/models/pseudonym.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
 - spec/dummy/app/models/submission.rb
@@ -520,6 +574,7 @@ files:
 - spec/dummy/db/migrate/20190927204546_create_admins.rb
 - spec/dummy/db/migrate/20200415171620_create_groups.rb
 - spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
+- spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
@@ -539,6 +594,19 @@ files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json
@@ -575,9 +643,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.2
 signing_key:
@@ -585,7 +653,6 @@ specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/fork_gather_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_checker_spec.rb
 - spec/canvas_sync/jobs/report_processor_job_spec.rb
@@ -637,6 +704,7 @@ test_files:
 - spec/dummy/app/models/enrollment.rb
 - spec/dummy/app/models/group.rb
 - spec/dummy/app/models/group_membership.rb
+- spec/dummy/app/models/pseudonym.rb
 - spec/dummy/app/models/role.rb
 - spec/dummy/app/models/section.rb
 - spec/dummy/app/models/submission.rb
@@ -699,6 +767,7 @@ test_files:
 - spec/dummy/db/migrate/20190927204546_create_admins.rb
 - spec/dummy/db/migrate/20200415171620_create_groups.rb
 - spec/dummy/db/migrate/20200416214248_create_group_memberships.rb
+- spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb
 - spec/dummy/db/schema.rb
 - spec/dummy/db/test.sqlite3
 - spec/dummy/log/development.log
@@ -718,6 +787,19 @@ test_files:
 - spec/factories/submission_factory.rb
 - spec/factories/term_factory.rb
 - spec/factories/user_factory.rb
+- spec/job_batching/batch_aware_job_spec.rb
+- spec/job_batching/batch_spec.rb
+- spec/job_batching/callback_spec.rb
+- spec/job_batching/flow_spec.rb
+- spec/job_batching/integration/integration.rb
+- spec/job_batching/integration/nested.rb
+- spec/job_batching/integration/simple.rb
+- spec/job_batching/integration/workflow.rb
+- spec/job_batching/integration_helper.rb
+- spec/job_batching/sidekiq_spec.rb
+- spec/job_batching/status_spec.rb
+- spec/job_batching/support/base_job.rb
+- spec/job_batching/support/sample_callback.rb
 - spec/spec_helper.rb
 - spec/support/fake_canvas.rb
 - spec/support/fixtures/canvas_responses/admins.json