canvas_sync 0.26.1 → 0.27.1.beta2

data/lib/canvas_sync/jobs/sync_rubric_associations_job.rb

@@ -1,15 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncRubricAssociationsJob < ReportStarter
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "rubric_associations_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::RubricAssociationsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncRubricAssociationsJob < ReportSyncTask
+      report_name "rubric_associations_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_rubrics_job.rb

@@ -1,15 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncRubricsJob < ReportStarter
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "rubrics_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::RubricsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncRubricsJob < ReportSyncTask
+      report_name "rubrics_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_scores_job.rb

@@ -1,18 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncScoresJob < ReportStarter
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_scores_data_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::NormalProcessor.to_s,
-          {
-            klass: 'Score',
-            mapping: 'scores'
-          },
-        )
-      end
+    class SyncScoresJob < ReportSyncTask
+      report_name "proserv_scores_data_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_submissions_job.rb

@@ -1,23 +1,14 @@
 module CanvasSync
   module Jobs
-    class SyncSubmissionsJob < ReportStarter
-      # Syncs Submissions
-      #
-      # Starts a report processor for the submission report
-      # (the proserv_student_submissions_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_student_submissions_csv",
-          merge_report_params(options, {
-            include_all: options[:include_all],
-            enrollment_batch_size: options[:enrollment_batch_size],
-            include_all_except: options[:include_all_except]
-          }, {}),
-          CanvasSync::Processors::SubmissionsProcessor.to_s,
-          {},
-        )
+    class SyncSubmissionsJob < ReportSyncTask
+      report_name "proserv_student_submissions_csv"
+
+      def report_parameters
+        merge_report_params(options, {
+          include_all: options[:include_all],
+          enrollment_batch_size: options[:enrollment_batch_size],
+          include_all_except: options[:include_all_except]
+        })
       end
     end
   end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -28,13 +28,15 @@ module CanvasSync
               end
 
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: 8) do |b|
-                b.description = "TermBatchJob(#{term_id}) Root"
+                b.description = "Term##{term_id} Fork"
                 b.context = local_context
                 b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
               end
             end
           else
-            JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context, concurrency: 8)
+            JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context, concurrency: 8) do |b|
+              b.description = "All Terms"
+            end
           end
         end
       end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.26.1".freeze
+  VERSION = "0.27.1.beta2".freeze
 end

data/lib/canvas_sync.rb

@@ -10,14 +10,12 @@ require "canvas_sync/api_syncable"
 require "canvas_sync/record"
 require "canvas_sync/live_events"
 require "canvas_sync/jobs/report_starter"
-require "canvas_sync/jobs/report_checker"
-require "canvas_sync/jobs/report_processor_job"
 require "canvas_sync/batch_processor"
 require "canvas_sync/config"
 
 require "joblin"
 
-Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
+Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/**/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/generators/*.rb"].each { |file| require file }
@@ -172,6 +170,11 @@ module CanvasSync
       options: {},
       **kwargs
     ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+      if CanvasSync::SyncBatch.where(batch_genre: 'beta_cleanup', status: 'processing').count > 0
+        Rails.logger.warn("There is currently a batch of genre 'beta_cleanup' in progress, skipping new provisioning sync to avoid conflicts.")
+        return
+      end
+
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -201,6 +204,7 @@ module CanvasSync
 
       root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
       concurrent_root_chain = JobBatches::ChainBuilder.new(JobBatches::ConcurrentBatchJob)
+      concurrent_root_chain.kwargs[:description] = "Default Concurrent Batch"
       root_chain << concurrent_root_chain
       current_chain = concurrent_root_chain
 
@@ -260,10 +264,33 @@ module CanvasSync
       if given_models.include?('pseudonyms')
         root_chain << Concerns::UserViaPseudonym::RefreshUserCachesJob
       end
-
       root_chain
     end
 
+    # wrapper around the default provisioning chain for the weekly beta refresh
+    def default_beta_cleanup_chain(models, term_scope: nil, term_scoped_models: DEFAULT_TERM_SCOPE_MODELS, options: {}, **kwargs)
+      updated_after = kwargs[:updated_after] || 2.weeks.ago
+
+      job_chain = default_provisioning_report_chain(
+        models,
+        term_scope: term_scope,
+        term_scoped_models: term_scoped_models,
+        batch_genre: 'beta_cleanup',
+        updated_after:,
+        options: options,
+        **kwargs
+      )
+      return if job_chain.nil?
+
+      # We add a job to create temporary tables at the start of the chain for each model
+      # and transfer data into it in order to compare it later with the re-synced data
+      job_chain.insert_at(0, { job: CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob.to_s, options: { models:, updated_after: } })
+      # >> CanvasSync default provisioning chain would be in this position HERE <<
+      job_chain.insert({ job: CanvasSync::Jobs::BetaCleanup::DeleteRelatedRecordsJob.to_s, options: { models: } })
+      job_chain.insert({ job: CanvasSync::Jobs::BetaCleanup::DeleteTempTablesJob.to_s, options: { models: } })
+      job_chain
+    end
+
     def base_canvas_sync_chain(
       legacy_support: false, # Import records 1 by 1 instead of with bulk upserts
       account_id: nil, # legacy/non PandaPal apps

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -30,7 +30,7 @@ RSpec.describe CanvasSync do
           provisioning: { c: 3 },
           global: { d: 4 },
         })
-        expect(chain.normalize!).to eq({
+        expect(chain.normalize!).to match({
           :job => "CanvasSync::Jobs::BeginSyncChainJob",
           :args => [
             [
@@ -51,7 +51,7 @@ RSpec.describe CanvasSync do
                     },
                   ],
                 ],
-                :kwargs=>{},
+                :kwargs=>{ description: String },
               },
             ],
           ],
@@ -62,7 +62,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing users with a term scope' do
         it 'syncs the users in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -82,7 +82,7 @@ RSpec.describe CanvasSync do
                     }
                   ]
                 ],
-                :kwargs=>{},
+                :kwargs=>{ description: String },
               }
             ]],
             :kwargs => {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil},
@@ -93,7 +93,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing users without a term scope' do
         it 'syncs users along with the rest of the provisioning report' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -109,7 +109,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil},
@@ -120,7 +120,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles with a term scope' do
         it 'syncs the roles in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -136,7 +136,7 @@ RSpec.describe CanvasSync do
                     }
                   },
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -147,7 +147,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles without a term scope' do
         it 'syncs roles separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -163,7 +163,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -172,7 +172,7 @@ RSpec.describe CanvasSync do
 
         it 'passes along options to the roles job' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], options: { roles: { show_inherited: true } } )
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -188,7 +188,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -200,7 +200,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins with a term scope' do
         it 'syncs the admins in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -216,7 +216,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -227,7 +227,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins without a term scope' do
         it 'syncs admins separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -243,7 +243,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -255,7 +255,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -274,7 +274,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -286,7 +286,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncSubmissionsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -305,7 +305,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -317,7 +317,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentGroupsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
             :args => [[
               {
@@ -336,7 +336,7 @@ RSpec.describe CanvasSync do
                     }
                   }
                 ]],
-                :kwargs=>{}
+                :kwargs=>{ description: String }
               }
             ]],
             :kwargs => {:batch_genre=>nil, :full_sync_every=>nil, :legacy_support=>false, :updated_after=>nil},
@@ -344,6 +344,46 @@ RSpec.describe CanvasSync do
         end
       end
     end
+    context "there is a beta cleanup batch in progress" do
+      it "does not start a provisioning sync" do
+        CanvasSync::SyncBatch.create!(batch_genre: "beta_cleanup", status: "processing")
+        expect(CanvasSync.default_beta_cleanup_chain(%w[users])).to be_nil
+      end
+    end
+  end
+
+  describe ".default_beta_cleanup_chain" do
+    it "creates a batch of genre 'beta_cleanup'" do
+      updated_after = 2.weeks.ago
+      chain = CanvasSync.default_beta_cleanup_chain(%w[users enrollments], updated_after:)
+      expect(chain.normalize!).to eq({
+        :job=>"CanvasSync::Jobs::BeginSyncChainJob",
+        :args=>
+          [[{:job=>"CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob",
+             :options=>
+              {:models=>["users", "enrollments"], :updated_after=>updated_after}},
+            {:job=>"Joblin::Batching::ConcurrentBatchJob",
+             :args=>
+              [[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
+                {:job=>"CanvasSync::Jobs::SyncTermsJob",
+                 :args=>[],
+                 :kwargs=>
+                  {:term_scope=>nil,
+                   :sub_jobs=>
+                    [{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob",
+                      :options=>{:models=>["enrollments"]}}]}}]],
+             :kwargs=>{:description=>"Default Concurrent Batch"}},
+            {:job=>"CanvasSync::Jobs::BetaCleanup::DeleteRelatedRecordsJob",
+             :options=>{:models=>["users", "enrollments"]}},
+            {:job=>"CanvasSync::Jobs::BetaCleanup::DeleteTempTablesJob",
+             :options=>{:models=>["users", "enrollments"]}}]],
+        :kwargs=>
+          {:legacy_support=>false,
+           :updated_after=>updated_after,
+           :full_sync_every=>nil,
+           :batch_genre=>"beta_cleanup"}
+      })
+    end
   end
 
   describe ".sync_scope" do

data/spec/canvas_sync/jobs/report_starter_spec.rb

@@ -4,69 +4,116 @@ RSpec.describe CanvasSync::Jobs::ReportStarter do
   let(:report_params) { { 'parameters[users]' => true } }
   let(:report_name) { 'provisioning_csv' }
   let(:processor) { 'CoolProcessor' }
-  let(:report_checker_double) { double }
   let(:options) { { models: ['users'] } }
 
   describe '#perform' do
-    it 'tells Canvas to start the report and then enqueues a ReportChecker with the report id' do
-      expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-        .with('self', report_name, report_params)
-        .and_return({ 'id' => 1 })
+    it 'delegates to LegacyReportShimTask with merged options' do
+      expect(CanvasSync::Jobs::LegacyReportShimTask).to receive(:perform_later) do |merged_options|
+        expect(merged_options[:models]).to eq(['users'])
+        expect(merged_options[:legacy_report_starter][:report]).to eq(report_name)
+        expect(merged_options[:legacy_report_starter][:params]).to eq(report_params)
+        expect(merged_options[:legacy_report_starter][:processor]).to eq(processor)
+      end
+
+      CanvasSync::Jobs::ReportStarter.perform_now(
+        report_name,
+        report_params,
+        processor,
+        options
+      )
+    end
+
+    it 'passes through allow_redownloads option' do
+      expect(CanvasSync::Jobs::LegacyReportShimTask).to receive(:perform_later)
+
+      CanvasSync::Jobs::ReportStarter.perform_now(
+        report_name,
+        report_params,
+        processor,
+        options,
+        allow_redownloads: true
+      )
+    end
+
+    it 'preserves original options' do
+      options_with_extras = options.merge({
+        account_id: '12345',
+        canvas_term_id: 99
+      })
 
-      expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
+      expect(CanvasSync::Jobs::LegacyReportShimTask).to receive(:perform_later) do |merged_options|
+        expect(merged_options[:account_id]).to eq('12345')
+        expect(merged_options[:canvas_term_id]).to eq(99)
+        expect(merged_options[:models]).to eq(['users'])
+      end
 
       CanvasSync::Jobs::ReportStarter.perform_now(
-                                report_name,
-                                report_params,
-                                processor,
-                                options)
+        report_name,
+        report_params,
+        processor,
+        options_with_extras
+      )
     end
+  end
+
+  describe 'LegacyReportShimTask' do
+    let(:legacy_options) do
+      {
+        models: ['users'],
+        legacy_report_starter: {
+          report: report_name,
+          params: report_params,
+          processor: processor
+        }
+      }
+    end
+
+    describe '#report_name' do
+      it 'extracts report name from legacy options' do
+        task = CanvasSync::Jobs::LegacyReportShimTask.new(legacy_options, {}, {})
+        expect(task.report_name).to eq(report_name)
+      end
+    end
+
+    describe '#report_parameters' do
+      it 'extracts report parameters from legacy options' do
+        task = CanvasSync::Jobs::LegacyReportShimTask.new(legacy_options, {}, {})
+        expect(task.report_parameters).to eq(report_params)
+      end
+    end
+
+    describe '#process' do
+      it 'calls the processor class with file, options, and account_id' do
+        processor_class = double('CoolProcessor')
+        stub_const('CoolProcessor', processor_class)
+
+        task = CanvasSync::Jobs::LegacyReportShimTask.new(legacy_options, {}, {})
+        allow(task).to receive(:batch_context).and_return({})
+
+        expect(processor_class).to receive(:process).with(
+          '/path/to/file.csv',
+          legacy_options,
+          'self'
+        )
+
+        task.process('/path/to/file.csv')
+      end
+
+      it 'uses account_id from options if present' do
+        processor_class = double('CoolProcessor')
+        stub_const('CoolProcessor', processor_class)
+
+        options_with_account = legacy_options.merge(account_id: '12345')
+        task = CanvasSync::Jobs::LegacyReportShimTask.new(options_with_account, {}, {})
+        allow(task).to receive(:batch_context).and_return({})
+
+        expect(processor_class).to receive(:process).with(
+          '/path/to/file.csv',
+          options_with_account,
+          '12345'
+        )
 
-    context 'allow_redownloads is true' do
-      # context 'the report has already been cached' do
-      #   it 'runs the report checker for the already started report' do
-      #     expect_any_instance_of(Bearcat::Client).to_not receive(:start_report)
-
-      #     expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_return(report_checker_double)
-      #     expect(report_checker_double).to receive(:perform_later).with(
-      #       report_name,
-      #       1,
-      #       processor,
-      #       options
-      #     )
-
-      #     CanvasSync::Jobs::ReportStarter.perform_now(
-      #                               report_name,
-      #                               report_params,
-      #                               processor,
-      #                               options,
-      #                               allow_redownloads: true
-      #     )
-      #   end
-      # end
-
-      context 'the report has not been started before' do
-        it 'starts a new report and caches the report id' do
-          expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-            .with('self', report_name, report_params)
-            .and_return({ 'id' => 1 })
-
-          expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_return(report_checker_double)
-          expect(report_checker_double).to receive(:perform_later).with(
-            report_name,
-            1,
-            processor,
-            options
-          )
-
-          CanvasSync::Jobs::ReportStarter.perform_now(
-                                    report_name,
-                                    report_params,
-                                    processor,
-                                    options,
-                                    allow_redownloads: true
-          )
-        end
+        task.process('/path/to/file.csv')
       end
     end
   end