canvas_sync 0.16.4 → 0.17.0.beta1

data/spec/dummy/app/models/pseudonym.rb

@@ -0,0 +1,14 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class Pseudonym < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
+end

data/spec/dummy/app/models/submission.rb

@@ -20,6 +20,7 @@ class Submission < ApplicationRecord
     canvas_user_id: :user_id,
     submitted_at: :submitted_at,
     graded_at: :graded_at,
+    cached_due_date: :due_at,
     score: :score,
     excused: :excused,
     workflow_state: :workflow_state,

data/spec/dummy/app/models/user.rb

@@ -11,6 +11,7 @@ class User < ApplicationRecord
   include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
+  has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admin_roles, through: :admins, source: :role

data/spec/dummy/config/environments/test.rb

@@ -37,6 +37,8 @@ Rails.application.configure do
   # Print deprecation notices to the stderr.
   config.active_support.deprecation = :stderr
 
+  config.active_job.queue_adapter = :test
+
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
 end

data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb

@@ -0,0 +1,24 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreatePseudonyms < ActiveRecord::Migration[5.1]
+  def change
+    create_table :pseudonyms do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_user_id
+      t.string :sis_id
+      t.string :unique_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :pseudonyms, :canvas_id, unique: true
+    add_index :pseudonyms, :canvas_user_id
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_04_16_214248) do
+ActiveRecord::Schema.define(version: 2020_10_18_210836) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -83,7 +83,7 @@ ActiveRecord::Schema.define(version: 2020_04_16_214248) do
     t.index ["canvas_id"], name: "index_assignments_on_canvas_id", unique: true
   end
 
-  create_table "canvas_sync_job_logs", force: :cascade do |t|
+  create_table "canvas_sync_job_logs", id: :serial, force: :cascade do |t|
     t.datetime "started_at"
     t.datetime "completed_at"
     t.string "exception"
@@ -92,13 +92,21 @@ ActiveRecord::Schema.define(version: 2020_04_16_214248) do
     t.string "status"
     t.text "metadata"
     t.text "job_arguments"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.datetime "created_at"
+    t.datetime "updated_at"
     t.string "job_id"
     t.integer "fork_count"
     t.index ["job_id"], name: "index_canvas_sync_job_logs_on_job_id"
   end
 
+  create_table "canvas_sync_sync_batches", id: :serial, force: :cascade do |t|
+    t.datetime "started_at"
+    t.datetime "completed_at"
+    t.string "status"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "context_module_items", force: :cascade do |t|
     t.bigint "canvas_id"
     t.bigint "canvas_context_module_id"
@@ -191,6 +199,18 @@ ActiveRecord::Schema.define(version: 2020_04_16_214248) do
     t.index ["canvas_id"], name: "index_groups_on_canvas_id", unique: true
   end
 
+  create_table "pseudonyms", force: :cascade do |t|
+    t.bigint "canvas_id", null: false
+    t.bigint "canvas_user_id"
+    t.string "sis_id"
+    t.string "unique_id"
+    t.string "workflow_state"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_id"], name: "index_pseudonyms_on_canvas_id", unique: true
+    t.index ["canvas_user_id"], name: "index_pseudonyms_on_canvas_user_id"
+  end
+
   create_table "roles", force: :cascade do |t|
     t.integer "canvas_id", null: false
     t.string "label"

data/spec/job_batching/batch_aware_job_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+RSpec.describe CanvasSync::JobBatches::BatchAwareJob do
+  include ActiveJob::TestHelper
+
+  after do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+
+  context "When Performing" do
+    context 'when without batch' do
+      it 'just yields' do
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_successful_job)
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:process_failed_job)
+        expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+
+        BatchTestJobBase.perform_now
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+
+      context 'when successful' do
+        it 'yields' do
+          expect_any_instance_of(BatchTestJobBase).to receive(:perform)
+          BatchTestJobBase.perform_now
+        end
+
+        it 'calls process_successful_job' do
+          job = BatchTestJobBase.new
+          job.instance_variable_set(:@bid, bid)
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(bid, job.job_id)
+          job.perform_now
+        end
+      end
+
+      context 'when failed' do
+        it 'calls process_failed_job and reraises exception' do
+          reraised = false
+          job = FailingBatchTestJobBase.new
+          job.instance_variable_set(:@bid, bid)
+          expect(CanvasSync::JobBatches::Batch).to receive(:process_failed_job)
+          begin
+            job.perform_now
+          rescue
+            reraised = true
+          end
+          expect(reraised).to be_truthy
+        end
+      end
+    end
+  end
+
+  context "When Enqueueing" do
+    context 'when without batch' do
+      it 'just yields' do
+        expect(CanvasSync::JobBatches::Batch).not_to receive(:increment_job_queue)
+        BatchTestJobBase.perform_later
+        expect(BatchTestJobBase).to have_been_enqueued
+      end
+    end
+
+    context 'when in batch' do
+      let(:bid) { 'SAMPLEBID' }
+
+      before do
+        Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid)
+        Thread.current[:batch].instance_variable_set(:@open, true)
+      end
+
+      it 'yields' do
+        expect {
+          BatchTestJobBase.perform_later
+        }.to enqueue_job(BatchTestJobBase)
+      end
+
+      it 'assigns bid to job metadata' do
+        job = BatchTestJobBase.perform_later
+        expect(job.bid).to eq bid
+        expect(job.serialize['batch_id']).to eq bid
+      end
+    end
+  end
+
+  context 'worker' do
+    it 'defines method bid' do
+      expect(ActiveJob::Base.instance_methods).to include(:bid)
+    end
+
+    it 'defines method batch' do
+      expect(ActiveJob::Base.instance_methods).to include(:batch)
+    end
+
+    it 'defines method valid_within_batch?' do
+      expect(ActiveJob::Base.instance_methods).to include(:valid_within_batch?)
+    end
+  end
+end

data/spec/job_batching/batch_spec.rb

@@ -0,0 +1,363 @@
+require 'spec_helper'
+
+class TestWorker < BatchTestJobBase
+  def perform
+  end
+end
+
+RSpec.describe CanvasSync::JobBatches::Batch do
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates bid when called without it' do
+      expect(subject.new.bid).not_to be_nil
+    end
+
+    it 'reuses bid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.bid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  describe '#description' do
+    let(:description) { 'custom description' }
+    before do
+      subject.description = description
+      subject.jobs { }
+    end
+
+    it 'sets descriptions' do
+      expect(subject.description).to eq(description)
+    end
+
+    it 'persists description' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'description') })
+        .to eq(description)
+    end
+  end
+
+  describe '#callback_queue' do
+    let(:callback_queue) { 'custom_queue' }
+    before do
+      subject.callback_queue = callback_queue
+      subject.jobs { }
+    end
+
+    it 'sets callback_queue' do
+      expect(subject.callback_queue).to eq(callback_queue)
+    end
+
+    it 'persists callback_queue' do
+      expect(CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{subject.bid}", 'callback_queue') })
+        .to eq(callback_queue)
+    end
+  end
+
+  describe '#jobs' do
+    it 'throws error if no block given' do
+      expect { subject.jobs }.to raise_error CanvasSync::JobBatches::Batch::NoBlockGivenError
+    end
+
+    it 'increments to_process (when started)'
+
+    it 'decrements to_process (when finished)'
+    # it 'calls process_successful_job to wait for block to finish' do
+    #   batch = CanvasSync::JobBatches::Batch.new
+    #   expect(CanvasSync::JobBatches::Batch).to receive(:process_successful_job).with(batch.bid)
+    #   batch.jobs {}
+    # end
+
+    it 'sets Thread.current bid' do
+      batch = CanvasSync::JobBatches::Batch.new
+      batch.jobs do
+        expect(Thread.current[:batch]).to eq(batch)
+      end
+    end
+  end
+
+  describe '#invalidate_all' do
+    class InvalidatableJob < BatchTestJobBase
+      def perform
+        return unless valid_within_batch?
+        was_performed
+      end
+
+      def was_performed; end
+    end
+
+    it 'marks batch in redis as invalidated' do
+      batch = CanvasSync::JobBatches::Batch.new
+      job = InvalidatableJob.new
+      allow(job).to receive(:was_performed)
+
+      batch.invalidate_all
+      batch.jobs { job.perform }
+
+      expect(job).not_to have_received(:was_performed)
+    end
+
+    context 'nested batches' do
+      let(:batch_parent) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_1) { CanvasSync::JobBatches::Batch.new }
+      let(:batch_child_2) { CanvasSync::JobBatches::Batch.new }
+      let(:job_of_parent) { InvalidatableJob.new }
+      let(:job_of_child_1) { InvalidatableJob.new }
+      let(:job_of_child_2) { InvalidatableJob.new }
+
+      before do
+        allow(job_of_parent).to receive(:was_performed)
+        allow(job_of_child_1).to receive(:was_performed)
+        allow(job_of_child_2).to receive(:was_performed)
+      end
+
+      it 'invalidates all job if parent batch is marked as invalidated' do
+        batch_parent.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).not_to have_received(:was_performed)
+        expect(job_of_child_1).not_to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+
+      it 'invalidates only requested batch' do
+        batch_child_2.invalidate_all
+        batch_parent.jobs do
+          [
+            job_of_parent.perform,
+            batch_child_1.jobs do
+              [
+                job_of_child_1.perform,
+                batch_child_2.jobs { job_of_child_2.perform }
+              ]
+            end
+          ]
+        end
+
+        expect(job_of_parent).to have_received(:was_performed)
+        expect(job_of_child_1).to have_received(:was_performed)
+        expect(job_of_child_2).not_to have_received(:was_performed)
+      end
+    end
+  end
+
+  describe '#process_failed_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      let(:failed_jid) { 'xxx' }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+      end
+
+      it 'add job to failed list' do
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, failed_jid)
+        failed = CanvasSync::JobBatches::Batch.redis { |r| r.smembers("BID-#{bid}-failed") }
+        expect(failed).to eq(['xxx', 'failed-job-id'])
+      end
+    end
+  end
+
+  describe '#process_successful_job' do
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+    let(:bid) { batch.bid }
+    let(:jid) { 'ABCD' }
+    before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
+
+    context 'complete' do
+      before { batch.on(:complete, Object) }
+      # before { batch.increment_job_queue(bid) }
+      before { batch.jobs do TestWorker.perform_async end }
+      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
+
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, 'failed-job-id')
+      end
+    end
+
+    context 'success' do
+      before { batch.on(:complete, Object) }
+      it 'tries to call complete callback' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+      end
+
+      it 'cleanups redis key' do
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+        expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
+      end
+    end
+  end
+
+  describe '#increment_job_queue' do
+    let(:bid) { 'BID' }
+    let(:batch) { CanvasSync::JobBatches::Batch.new }
+
+    it 'increments pending' do
+      batch.jobs do TestWorker.perform_async end
+      pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
+      expect(pending).to eq('1')
+    end
+
+    it 'increments total' do
+      batch.jobs do TestWorker.perform_async end
+      total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
+      expect(total).to eq('1')
+    end
+  end
+
+  describe '#enqueue_callbacks' do
+    let(:callback) { double('callback') }
+    let(:event) { :complete }
+
+    context 'on :success' do
+      let(:event) { :success }
+      context 'when no callbacks are defined' do
+        it 'clears redis keys' do
+          batch = CanvasSync::JobBatches::Batch.new
+          expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
+          CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+        end
+      end
+    end
+
+    context 'when already called' do
+      it 'returns and does not enqueue callbacks' do
+        batch = CanvasSync::JobBatches::Batch.new
+        batch.on(event, SampleCallback)
+        CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{batch.bid}", event, true) }
+
+        expect(batch).not_to receive(:push_callbacks)
+        CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+      end
+    end
+
+    context 'With ActiveJob Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback2', event.to_s, opts2, batch.bid, nil
+            )
+            expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
+              'SampleCallback', event.to_s, opts, batch.bid, nil
+            )
+          end
+        end
+      end
+    end
+
+    context 'With Sidekiq Adapter' do
+      around(:all) do |block|
+        CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
+        CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
+        block.run
+      end
+
+      context 'when not yet called' do
+        context 'when there is no callback' do
+          it 'it returns' do
+            batch = CanvasSync::JobBatches::Batch.new
+
+            expect(batch).not_to receive(:push_callbacks)
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when callback defined' do
+          let(:opts) { { 'a' => 'b' } }
+
+          it 'calls it passing options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+
+        context 'when multiple callbacks are defined' do
+          let(:opts) { { 'a' => 'b' } }
+          let(:opts2) { { 'b' => 'a' } }
+
+          it 'enqueues each callback passing their options' do
+            batch = CanvasSync::JobBatches::Batch.new
+            batch.on(event, SampleCallback, opts)
+            batch.on(event, SampleCallback2, opts2)
+
+            expect(Sidekiq::Client).to receive(:push_bulk).with(
+              'class' => Sidekiq::Batch::Callback::Worker,
+              'args' => [
+                ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
+                ['SampleCallback', event.to_s, opts, batch.bid, nil]
+              ],
+              'queue' => 'default'
+            )
+
+            CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
+          end
+        end
+      end
+    end
+  end
+end