canvas_sync 0.27.1.beta3 → 0.27.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 569b75049c7212d44318075a8d2ff1680cbd0d7d68a5446a0b94ec09718eef95
-  data.tar.gz: 911fd5399e951f4badf0dd77baf78badb8e930606e6f031bbda67eee79e05e72
+  metadata.gz: 778924d69dbb2a1f29a6e9b5101851ebda4042536949619b1aa5a2ebdd0f1f69
+  data.tar.gz: 06ce5ef0c498612f864dd54f817cb24200b9f64d6dcce644ab235ba784bd4d79
 SHA512:
-  metadata.gz: 1fc6af99ed8eb1e3f9c4b5be559beff9e390b9913c966641c5fa0ef4e77714494bc33b96b174d4c0b332bf1e288a566b9346569932e52760a7a501b520e65a57
-  data.tar.gz: b2ad6cbeb8b979ea843fab8ee161a4b7acc82b620fee99467284ff79e150e2625913c4797d057d76e253eb576e85c0c8e4368f9028a0660fcc2b80bfb29ec132
+  metadata.gz: 571f4296981c5fd72386d6b58d68864f719340da7862b4a5720a5d21d7ba5d146c23e1cfaf2f88b33c55bf80b0b02dd174a22e49519eecea8793b82d7c95b883
+  data.tar.gz: 484661a54a3b0fbeee189b9ee64e9554e6baa455da0ef3ed641327c71ba7cfbb40ddcab8394c23f4992f6d98447afe5c7abf58c56f225ef0fdf2689f7f8dfb1c

data/app/controllers/canvas_sync/api/v1/live_events_controller.rb

@@ -111,7 +111,6 @@ module CanvasSync::Api
       end
 
       def switch_tenant(&block)
-        # TODO Move detection of this param into the PandaPal Apartment Elevator
         if defined?(PandaPal) && (org = params[:organization] || params[:org]).present?
           org = PandaPal::Organization.find(org)
           if org.respond_to?(:switch_tenant)

data/lib/canvas_sync/engine.rb

@@ -45,10 +45,10 @@ module CanvasSync
     }
 
     initializer 'canvas_sync.global_methods' do
-      next if defined?(canvas_sync_client)
-
       require 'panda_pal'
 
+      next if defined?(canvas_sync_client)
+
       class ::Object
         def canvas_sync_client(account_id = nil)
           org = PandaPal::Organization.current
@@ -88,6 +88,16 @@ module CanvasSync
             end
           end
         end
+
+        if PandaPal::Organization.respond_to?(:after_panda_pal_interactive_install)
+          PandaPal::Organization.after_panda_pal_interactive_install do
+            ReportsMap.pretty_print_required_reports
+            puts ""
+            PandaPal::ConsoleHelpers.pause
+          rescue => ex
+            puts "[CanvasSync] Failed to print required reports: #{ex}"
+          end
+        end
       end
     rescue LoadError
     end

data/lib/canvas_sync/generators/templates/migrations/create_enrollments.rb

@@ -14,6 +14,7 @@ class CreateEnrollments < ActiveRecord::Migration[5.1]
       t.string :workflow_state
       t.string :base_role_type
       t.datetime :completed_at
+      t.boolean :restricted_access
 
       t.timestamps
     end

data/lib/canvas_sync/jobs/report_sync_task.rb

@@ -59,14 +59,30 @@ module CanvasSync
           { parameters: params }
         end
 
-        def report_name(n)
-          define_method(:report_name) { n }
+        def report_name(n = nil)
+          if n.present?
+            const_set("REPORT_NAME", n)
+            define_method(:report_name) { n }
+          else
+            const_get("REPORT_NAME") rescue nil
+          end
         end
 
-        def model(m)
-          define_method(:process) do |file|
-            m = m.constantize if m.is_a?(String) || m.is_a?(Symbol)
-            do_bulk_import(file, m)
+        def model(m = nil)
+          if m.present?
+            const_set("MODEL", m)
+          else
+            mdl = const_get("MODEL") rescue nil
+
+            mdl ||= begin
+              mbits = self.to_s.scan(AUTO_MODEL_REGEX)
+              if (m = mbits.last) && m[0].present?
+                model_name = m[0].singularize
+                model_name
+              end
+            end
+
+            mdl
           end
         end
 
@@ -85,7 +101,11 @@ module CanvasSync
       end
 
       def report_name
-        raise NotImplementedError
+        if defined?(self.class::REPORT_NAME)
+          self.class::REPORT_NAME
+        else
+          raise NotImplementedError
+        end
       end
 
       def report_parameters
@@ -95,13 +115,7 @@ module CanvasSync
       AUTO_MODEL_REGEX = /Sync(\w+)Job/
 
       def process(file)
-        mbits = self.class.to_s.scan(AUTO_MODEL_REGEX)
-        if (m = mbits.last) && m[0].present?
-          model_name = m[0].singularize
-          model = model_name.safe_constantize
-          return do_bulk_import(file, model) if model.present?
-        end
-        raise NotImplementedError
+        do_bulk_import(file, model_class)
       end
 
       def check_frequency
@@ -118,6 +132,14 @@ module CanvasSync
 
       protected
 
+      def model_class
+        mdl = self.class.model
+        mdl = mdl.constantize if mdl.is_a?(String) || mdl.is_a?(Symbol)
+        mdl
+      rescue NameError
+        raise NotImplementedError, "Model #{self.class.model} does not exist"
+      end
+
       def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil, &blk)
         Processors::ReportProcessor.new().do_bulk_import(report_file_path, model, options: options, mapping_key: mapping_key, &blk)
       end

data/lib/canvas_sync/processors/model_mappings.yml

@@ -165,6 +165,9 @@ enrollments:
     completed_at:
       database_column_name: completed_at
       type: datetime
+    restricted_access:
+      database_column_name: restricted_access
+      type: boolean
 
 sections:
   conflict_target: canvas_section_id

data/lib/canvas_sync/reports_map.rb

@@ -0,0 +1,130 @@
+module CanvasSync
+  module ReportsMap
+    REPORTS = {
+      "proservices_provisioning_csv" => {
+        human_name: "Proserv Provisioning Report",
+        module_name: "CustomReports::Proservices::Provisioning",
+      },
+      "proserv_assignment_group_export_csv" => {
+        human_name: "Assignment Group Export",
+        module_name: "CustomReports::Proserv",
+      },
+      "proserv_assignment_overrides_csv" => {
+        human_name: "Assignment Overrides Export",
+        module_name: "CustomReports::Proservices::AssignmentOverrides",
+      },
+      "proserv_content_migrations_csv" => {
+        human_name: "Professional Services Content Migrations Report",
+        module_name: "CustomReports::Proservices::ContentMigrations",
+      },
+      "proserv_context_module_items_csv" => {
+        human_name: "Professional Services Context Module Items Report",
+        module_name: "CustomReports::Proservices::ContextModuleItems",
+      },
+      "proserv_context_modules_csv" => {
+        human_name: "Professional Services Context Modules Report",
+        module_name: "CustomReports::Proservices::ContextModules",
+      },
+      "proserv_course_completion_csv" => {
+        human_name: "Student Course Completion Dates",
+        module_name: "CustomReports::Proserv",
+      },
+
+      "rubrics_csv" => {
+        human_name: "CustomReports::Rubrics::Rubrics",
+        module_name: "Rubrics Export Report",
+      },
+      "rubric_associations_csv" => {
+        human_name: "Rubrics Associations Export Report",
+        module_name: "CustomReports::Rubrics::RubricAssociations",
+      },
+      "rubric_assessments_csv" => {
+        human_name: "Rubrics Assessments Export Report",
+        module_name: "CustomReports::Rubrics::RubricAssessments",
+      },
+
+      "proserv_scores_data_csv" => {
+        human_name: "Professional Services Scores Data Report",
+        module_name: "CustomReports::Proservices::ScoresData",
+      },
+      "proserv_student_submissions_csv" => {
+        human_name: "Student Submissions",
+        module_name: "CustomReports::Proserv",
+      },
+    }
+
+    def self.define_report(name, human_name: nil, module_name: nil)
+      REPORTS[name] ||= {}
+      REPORTS[name][:human_name] = human_name if human_name
+      REPORTS[name][:module_name] = module_name if module_name
+    end
+
+    EXPLICIT_REQUIRED_REPORTS = Set.new
+
+    def self.require_report!(key, **kwargs)
+      define_report(key, **kwargs)
+      EXPLICIT_REQUIRED_REPORTS << key
+    end
+
+    def self.model_reports
+      mapping = {}
+
+      Jobs::ReportSyncTask.subclasses.each do |cls|
+        model = cls.model.to_s
+        report = cls.report_name
+        next unless model.present?
+        mapping[model] ||= []
+        mapping[model] << report
+      end
+
+      mapping
+    end
+
+    def self.required_reports
+      req = Set.new
+      req << "proservices_provisioning_csv"
+      req.merge(EXPLICIT_REQUIRED_REPORTS)
+
+      model_reports.each do |model, reports|
+        model_class = model.safe_constantize
+        next unless model_class.present?
+
+        req.merge(reports)
+      end
+
+      req
+    end
+
+    def self.pretty_print_required_reports(client = nil)
+      require 'colorized_string'
+      client ||= canvas_sync_client rescue nil
+
+      req_reports = required_reports
+      return "[CanvasSync] No reports required... supposedly" if req_reports.empty?
+
+      enabled_reports = client.report_list('self').map{|r| r["report"] }.to_set rescue nil
+
+      puts "[CanvasSync] Required Custom Reports:"
+      req_reports.each do |report|
+        color = enabled_reports.nil? ? :cyan : (enabled_reports.include?(report) ? :green : :red)
+        puts "- #{ColorizedString[report].colorize(color)}"
+        if (meta = REPORTS[report]) && (meta[:human_name] || meta[:module_name])
+          puts "    #{meta[:human_name]} (#{meta[:module_name]})"
+        end
+      end
+      puts "- #{ColorizedString["NB: Double check if you know this app requires non-CanvasSync provided models."].italic}"
+      puts "    #{ColorizedString["Non-standard reports must be registered with `CanvasSync.require_report!` from an initializer to be included in this list."].italic}"
+
+      puts ""
+
+      bits = req_reports.map do |report|
+        "settings[#{report}]=checked"
+      end
+
+      base_url = client.config[:prefix] rescue "<CANVAS URL>/"
+      plugins_url = "#{base_url}/plugins/account_reports".gsub(/\/+/, "/")
+      plugins_url += "?#{bits.join("&")}" if bits.any?
+      puts "Pre-filled link: " + ColorizedString[plugins_url].cyan
+    end
+  end
+end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.27.1.beta3".freeze
+  VERSION = "0.27.3".freeze
 end

data/lib/canvas_sync.rb

@@ -12,10 +12,11 @@ require "canvas_sync/live_events"
 require "canvas_sync/jobs/report_starter"
 require "canvas_sync/batch_processor"
 require "canvas_sync/config"
+require "canvas_sync/reports_map"
 
 require "joblin"
 
-Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/**/*.rb"].each { |file| require file }
+Dir[File.dirname(__FILE__) +  "/canvas_sync/jobs/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/processors/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/importers/*.rb"].each { |file| require file }
 Dir[File.dirname(__FILE__) +  "/canvas_sync/generators/*.rb"].each { |file| require file }
@@ -127,6 +128,10 @@ module CanvasSync
       default_provisioning_report_chain(models, **kwargs).process!
     end
 
+    def require_report!(*args, **kwargs)
+      ReportsMap.require_report!(*args, **kwargs)
+    end
+
     # Given a Model or Relation, scope it down to items that should be synced
     def sync_scope(scope, fallback_scopes: [])
       terms = [
@@ -170,11 +175,6 @@ module CanvasSync
       options: {},
       **kwargs
     ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
-      if CanvasSync::SyncBatch.where(batch_genre: 'beta_cleanup', status: 'processing').count > 0
-        Rails.logger.warn("There is currently a batch of genre 'beta_cleanup' in progress, skipping new provisioning sync to avoid conflicts.")
-        return
-      end
-
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -264,31 +264,8 @@ module CanvasSync
       if given_models.include?('pseudonyms')
         root_chain << Concerns::UserViaPseudonym::RefreshUserCachesJob
       end
-      root_chain
-    end
 
-    # wrapper around the default provisioning chain for the weekly beta refresh
-    def default_beta_cleanup_chain(models, term_scope: nil, term_scoped_models: DEFAULT_TERM_SCOPE_MODELS, options: {}, **kwargs)
-      updated_after = kwargs[:updated_after] || 2.weeks.ago
-
-      job_chain = default_provisioning_report_chain(
-        models,
-        term_scope: term_scope,
-        term_scoped_models: term_scoped_models,
-        batch_genre: 'beta_cleanup',
-        updated_after:,
-        options: options,
-        **kwargs
-      )
-      return if job_chain.nil?
-
-      # We add a job to create temporary tables at the start of the chain for each model
-      # and transfer data into it in order to compare it later with the re-synced data
-      job_chain.insert_at(0, { job: CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob.to_s, options: { models:, updated_after: } })
-      # >> CanvasSync default provisioning chain would be in this position HERE <<
-      job_chain.insert({ job: CanvasSync::Jobs::BetaCleanup::DeleteRelatedRecordsJob.to_s, options: { models: } })
-      job_chain.insert({ job: CanvasSync::Jobs::BetaCleanup::DeleteTempTablesJob.to_s, options: { models: } })
-      job_chain
+      root_chain
     end
 
     def base_canvas_sync_chain(

data/lib/tasks/canvas_sync_tasks.rake

@@ -0,0 +1,6 @@
+namespace :canvas_sync do
+  desc 'List the reports that CanvasSync requires to be present in the Canvas instance for it to operate correctly, based on the currently enabled features and models.'
+  task reports: :environment do
+    CanvasSync::ReportsMap.pretty_print_required_reports
+  end
+end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -344,46 +344,6 @@ RSpec.describe CanvasSync do
         end
       end
     end
-    context "there is a beta cleanup batch in progress" do
-      it "does not start a provisioning sync" do
-        CanvasSync::SyncBatch.create!(batch_genre: "beta_cleanup", status: "processing")
-        expect(CanvasSync.default_beta_cleanup_chain(%w[users])).to be_nil
-      end
-    end
-  end
-
-  describe ".default_beta_cleanup_chain" do
-    it "creates a batch of genre 'beta_cleanup'" do
-      updated_after = 2.weeks.ago
-      chain = CanvasSync.default_beta_cleanup_chain(%w[users enrollments], updated_after:)
-      expect(chain.normalize!).to eq({
-        :job=>"CanvasSync::Jobs::BeginSyncChainJob",
-        :args=>
-          [[{:job=>"CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob",
-             :options=>
-              {:models=>["users", "enrollments"], :updated_after=>updated_after}},
-            {:job=>"Joblin::Batching::ConcurrentBatchJob",
-             :args=>
-              [[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
-                {:job=>"CanvasSync::Jobs::SyncTermsJob",
-                 :args=>[],
-                 :kwargs=>
-                  {:term_scope=>nil,
-                   :sub_jobs=>
-                    [{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob",
-                      :options=>{:models=>["enrollments"]}}]}}]],
-             :kwargs=>{:description=>"Default Concurrent Batch"}},
-            {:job=>"CanvasSync::Jobs::BetaCleanup::DeleteRelatedRecordsJob",
-             :options=>{:models=>["users", "enrollments"]}},
-            {:job=>"CanvasSync::Jobs::BetaCleanup::DeleteTempTablesJob",
-             :options=>{:models=>["users", "enrollments"]}}]],
-        :kwargs=>
-          {:legacy_support=>false,
-           :updated_after=>updated_after,
-           :full_sync_every=>nil,
-           :batch_genre=>"beta_cleanup"}
-      })
-    end
   end
 
   describe ".sync_scope" do

data/spec/factories/account_factory.rb

@@ -1,6 +1,6 @@
 FactoryBot.define do
   factory :account do
-    sequence(:canvas_id) { |n| n }
+    canvas_id { SecureRandom.random_number(100_000_000) }
     sis_id { SecureRandom.hex }
     canvas_parent_account_id { 1 }
     sis_parent_account_id { SecureRandom.hex }

data/spec/support/fixtures/reports/enrollments.csv

@@ -1,3 +1,3 @@
-canvas_enrollment_id,canvas_course_id,course_id,canvas_user_id,user_id,role,role_id,canvas_section_id,section_id,status,canvas_associated_user_id,associated_user_id,created_by_sis,base_role_type,limit_section_privileges
-1,1,sis_MUSC_2520_834,2,sis_dalvarez_247,teacher,4,1,sis_section_MUSC_2520_752,active,,,false,TeacherEnrollment,false
-2,1,sis_MUSC_2520_834,3,sis_jweaver_720,student,4,1,sis_section_MUSC_2520_752,active,,,false,StudentEnrollment,false
+canvas_enrollment_id,canvas_course_id,course_id,canvas_user_id,user_id,role,role_id,canvas_section_id,section_id,status,canvas_associated_user_id,associated_user_id,created_by_sis,base_role_type,limit_section_privileges,restricted_access
+1,1,sis_MUSC_2520_834,2,sis_dalvarez_247,teacher,4,1,sis_section_MUSC_2520_752,active,,,false,TeacherEnrollment,false,false
+2,1,sis_MUSC_2520_834,3,sis_jweaver_720,student,4,1,sis_section_MUSC_2520_752,active,,,false,StudentEnrollment,false,true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.27.1.beta3
+  version: 0.27.3
 platform: ruby
 authors:
 - Instructure CustomDev
@@ -421,6 +421,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 email:
 - pseng@instructure.com
 executables: []
@@ -534,9 +554,6 @@ files:
 - lib/canvas_sync/importers/legacy_importer.rb
 - lib/canvas_sync/job.rb
 - lib/canvas_sync/jobs/begin_sync_chain_job.rb
-- lib/canvas_sync/jobs/beta_cleanup/create_temp_tables_job.rb
-- lib/canvas_sync/jobs/beta_cleanup/delete_related_records_job.rb
-- lib/canvas_sync/jobs/beta_cleanup/delete_temp_tables_job.rb
 - lib/canvas_sync/jobs/canvas_process_waiter.rb
 - lib/canvas_sync/jobs/report_starter.rb
 - lib/canvas_sync/jobs/report_sync_task.rb
@@ -567,12 +584,11 @@ files:
 - lib/canvas_sync/processors/normal_processor.rb
 - lib/canvas_sync/processors/report_processor.rb
 - lib/canvas_sync/record.rb
+- lib/canvas_sync/reports_map.rb
 - lib/canvas_sync/sidekiq_job.rb
 - lib/canvas_sync/version.rb
+- lib/tasks/canvas_sync_tasks.rake
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/create_temp_tables_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/delete_related_records_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/delete_temp_tables_spec.rb
 - spec/canvas_sync/jobs/canvas_process_waiter_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_starter_spec.rb
@@ -766,9 +782,6 @@ specification_version: 4
 summary: Gem for generating Canvas models and migrations and syncing data from Canvas
 test_files:
 - spec/canvas_sync/canvas_sync_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/create_temp_tables_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/delete_related_records_spec.rb
-- spec/canvas_sync/jobs/beta_cleanup/delete_temp_tables_spec.rb
 - spec/canvas_sync/jobs/canvas_process_waiter_spec.rb
 - spec/canvas_sync/jobs/job_spec.rb
 - spec/canvas_sync/jobs/report_starter_spec.rb

data/lib/canvas_sync/jobs/beta_cleanup/create_temp_tables_job.rb

@@ -1,30 +0,0 @@
-module CanvasSync
-  module Jobs::BetaCleanup
-    # This job creates "temporary" tables that holds the records that were updated since a given date
-    # and deletes those records from the main tables.
-    # The tables are actual tables and not the temporary tables in the DB sense
-    # because they need to be reused in other contexts.
-    # They are deleted in a different job at a later point
-    class CreateTempTablesJob < CanvasSync::Job
-      def perform(options = {})
-        canvas_tables = options[:models]
-
-        return if canvas_tables.empty?
-
-        canvas_tables.each do |table_name|
-          model = table_name.singularize.camelize.constantize
-
-          updated_after = options[:updated_after] || 2.weeks.ago
-          ActiveRecord::Base.connection.drop_table("beta_#{table_name}", if_exists: true)
-          ActiveRecord::Base.connection.exec_query(<<~SQL
-            CREATE TABLE beta_#{table_name} AS
-            SELECT * FROM #{model.quoted_table_name} WHERE updated_at >= '#{updated_after}';
-          SQL
-                                                  )
-          model.where("updated_at >= '#{updated_after}'").delete_all
-        end
-      end
-    end
-  end
-end
-

data/lib/canvas_sync/jobs/beta_cleanup/delete_related_records_job.rb

@@ -1,125 +0,0 @@
-module CanvasSync
-  module Jobs::BetaCleanup
-    class DeleteRelatedRecordsJob < CanvasSync::Job
-      # This method assumes that the main tables have been dumped into temporary tables (until provided date range, e.g 2 weeks)
-      # and that the main tables have been re-synced
-      # Considering these two criteria, assuming we have the main table now with
-      # records [1, 2, 3] and the temp table with records [1, 2, 3, 4], this means that
-      # record '4' needs to be taken out of related tables.
-      # This could for example be a record with a 'canvas_user_id = 4' in a table that is not part of CanvasSync models
-      # When this is implemented in CanvasSync, we could store the cleanup id and prefix the table name with it
-      # e.g: beta_cleanup_1_users
-      def delete_matching_records_between_main_and_temp
-        @canvas_tables.each do |table_name|
-          model = table_name.singularize.camelize.constantize
-
-          # here we can safely use the id to find the records we want - the primary keys are preserved in the temporary table
-          ActiveRecord::Base.connection.exec_query(<<~SQL
-            DELETE FROM beta_#{table_name} WHERE canvas_id IN (SELECT canvas_id FROM #{model.quoted_table_name});
-          SQL
-                                                  )
-        end
-      end
-
-      # possible foreign keys per table. Some LTI tables may or may not always use the Canvas ID
-      #
-      # Some tables like user_observers have multiple user_id in them (e.g 'observed_user_id' and 'observing_user_id')
-      # Not sure if this matters here, to investigate
-      def foreign_keys
-        @foreign_keys ||= @canvas_tables.map do |table_name|
-          model_name = table_name.singularize
-          fks = ["#{model_name}_id", "canvas_#{model_name}_id"]
-          fks.each { |fk| foreign_key_to_table[fk] = table_name }
-          fks
-        end.flatten
-      end
-
-      # Find better names for these next two methods
-      # fk => table_name (1:1)
-      def foreign_key_to_table
-        @foreign_key_to_table ||= {}.with_indifferent_access
-      end
-
-      # fk => table_name[] (1:many)
-      def foreign_key_to_tables
-        h = {}.with_indifferent_access
-        ActiveRecord::Base.connection.tables.map do |lti_table|
-          # ignore the models we just synced or the beta tables
-          # This may not be super reliable in case someone calls a table 'beta_xyz'
-          next if @canvas_tables.include?(lti_table.gsub(/^beta_/, ''))
-
-          columns = ActiveRecord::Base.connection.columns(lti_table).map(&:name)
-          cols_in_table = foreign_keys & columns
-
-          next if cols_in_table.empty?
-
-          cols_in_table.map do |col|
-            h[col] ||= []
-            h[col] << lti_table
-          end
-        end
-        @foreign_key_to_tables ||= h
-      end
-
-      def active_tables
-        # check the count of each "beta_" table
-        active_tables_sql = @canvas_tables.map do |table|
-          "SELECT 'beta_#{table}' AS table_name FROM beta_#{table} GROUP BY table_name HAVING COUNT(*) > 0"
-        end.join(' UNION ALL ')
-
-        # active tables are tables with at least one record
-        # If a beta table has no rows, we can ignore it in the next step
-        active_tables = ActiveRecord::Base.connection.exec_query(active_tables_sql).rows.flatten
-      end
-
-      def perform(options = {})
-        canvas_tables = options[:models] || []
-        @canvas_tables = canvas_tables
-
-        return [] if canvas_tables.empty?
-
-        delete_matching_records_between_main_and_temp
-
-        # every table is cleaned by foreign key to be idempotent in case of job failures (instead of cleaned by table)
-        # we can not clean by table because multiple tables may be using the same 'user_id' foreign key for example
-        foreign_key_to_tables.each do |fk, table_names|
-          temp_related_table = "beta_#{foreign_key_to_table[fk]}"
-          next unless active_tables.include?(temp_related_table)
-
-          pk = fk.include?('canvas') ? 'canvas_id' : 'id'
-
-          # Ideally this commented CTE is used so this process is idempotent but there is an issue when there are two tables with, for example user_id & canvas_user_id respectively
-          # The user_id foreign key will be processed and will delete the temporary records, which means the second foreign key, canvas_user_id, will not be able to retrive any records from it
-          # I don't think idempotency is such a big deal here because:
-          #   1) this runs only in beta instances
-          #   2) it runs for data within (most likely) the last 2 weeks, which means not that many records
-          # In the meantime, we can use the CTE that does not delete records
-=begin
-          sql = <<~SQL
-            WITH stale_records AS (
-              DELETE FROM #{temp_related_table}
-              WHERE #{pk} IN (
-                SELECT #{pk} FROM #{temp_related_table} ORDER BY #{pk} LIMIT 1000
-              )
-              RETURNING #{pk}
-            ),
-          SQL
-=end
-          sql = <<~SQL
-            WITH stale_records AS (
-              SELECT #{pk} FROM #{temp_related_table} ORDER BY #{pk} LIMIT 1000
-            ),
-          SQL
-
-          # using CTEs for multiple deletion statements in one query
-          # This allows using the same 'stale_records' CTE for all of the delete statements
-          sql << table_names.map { |table_name| "#{table_name}_del AS ( DELETE FROM #{table_name} WHERE #{fk} IN (SELECT #{pk} FROM stale_records) )" }.join(",\n")
-          sql << 'SELECT count(*) AS deleted_count FROM stale_records'
-          ActiveRecord::Base.transaction do # transaction to use the CTE multiple times
-            ActiveRecord::Base.connection.exec_query(sql)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/canvas_sync/jobs/beta_cleanup/delete_temp_tables_job.rb

@@ -1,15 +0,0 @@
-module CanvasSync
-  module Jobs::BetaCleanup
-    class DeleteTempTablesJob < CanvasSync::Job
-      def perform(options = {})
-        tables = options[:models] || []
-        return if tables.empty?
-
-        tables.each do |table_name|
-          ActiveRecord::Base.connection.drop_table("beta_#{table_name}", if_exists: true)
-        end
-      end
-    end
-  end
-end
-

data/spec/canvas_sync/jobs/beta_cleanup/create_temp_tables_spec.rb

@@ -1,22 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob do
-  describe "#perform" do
-    it "creates a beta_<table> for each model with the records with an updated_at after the updated_after param " do
-      MODELS_TO_SYNC = %w[accounts courses enrollments users]
-
-      MODELS_TO_SYNC.each do |table_name|
-        model = table_name.singularize.to_sym
-        FactoryBot.create_list(model, 3, updated_at: 4.weeks.ago)
-        FactoryBot.create_list(model, 2, updated_at: 1.week.ago)
-      end
-
-      described_class.new.perform({ models: MODELS_TO_SYNC, updated_after: 2.weeks.ago })
-      beta_tables = MODELS_TO_SYNC.map { |table_name| "beta_#{table_name}" }
-      expect(ActiveRecord::Base.connection.tables & beta_tables).to match_array(beta_tables)
-      expect(beta_tables.all? { |table_name| ActiveRecord::Base.connection.exec_query("SELECT COUNT(*) FROM #{table_name}").rows.flatten.first == 2 }).to be true
-      # original tables should no longer have the records that were moved to the beta tables
-      expect(MODELS_TO_SYNC.all? { |table_name| ActiveRecord::Base.connection.exec_query("SELECT COUNT(*) FROM #{table_name}").rows.flatten.first == 3 }).to be true
-    end
-  end
-end

data/spec/canvas_sync/jobs/beta_cleanup/delete_related_records_spec.rb

@@ -1,56 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe CanvasSync::Jobs::BetaCleanup::DeleteRelatedRecordsJob do
-  describe "#perform" do
-    let(:models_to_sync) { %w[accounts] }
-    let(:conn) { ActiveRecord::Base.connection }
-
-    before(:each) do
-      @accounts = FactoryBot.create_list(:account, 3, updated_at: 1.week.ago)
-      @records_to_readd = @accounts[0..1]
-      CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob.new.perform({ models: models_to_sync, updated_after: 2.weeks.ago })
-      # we pretend a sync happens, which re-adds 2/3 of the previous records
-      Account.create!(@records_to_readd.map(&:attributes))
-      # This means that records[2] should have its association deleted in any LTI specific tables via its foreign key
-    end
-
-    it "deletes records in associated tables via their foreign key" do
-      conn.exec_query(<<~SQL)
-        CREATE TABLE purchases (
-          id SERIAL PRIMARY KEY,
-          canvas_account_id INTEGER
-        )
-      SQL
-      conn.exec_query(<<~SQL)
-        INSERT INTO purchases (canvas_account_id) VALUES (#{@accounts[0].canvas_id}), (#{@accounts[1].canvas_id}), (#{@accounts[2].canvas_id});
-      SQL
-      # the record in this table with canvas_account_id = 3 should be deleted (records[2])
-      described_class.new.perform({ models: models_to_sync })
-      expect(conn.exec_query("SELECT canvas_account_id FROM purchases").rows.flatten).to match_array(@records_to_readd.map(&:canvas_id))
-    end
-
-    # BACKWARD COMPABILITY WITH THE ID COLUMN
-    # In newer versions of CanvasSync, tables no longer have the "id" primary key and only have the "canvas_id" column
-    # Older apps may still have tables with an "id" primary key
-    context "when the table has an 'id' column as a primary key instead of 'canvas_id'" do
-      it "deletes the records via the 'account_id' foreign key" do
-        conn.execute(<<~SQL)
-          ALTER TABLE accounts ADD COLUMN id SERIAL PRIMARY KEY;
-          ALTER TABLE beta_accounts ADD COLUMN id SERIAL PRIMARY KEY;
-        SQL
-        conn.exec_query(<<~SQL)
-          CREATE TABLE customers (
-            id SERIAL PRIMARY KEY,
-            account_id INTEGER
-          )
-        SQL
-        conn.exec_query(<<~SQL)
-          INSERT INTO customers (account_id) VALUES (1), (2), (3);
-        SQL
-        described_class.new.perform({ models: models_to_sync })
-        expect(conn.exec_query("SELECT account_id FROM customers").rows.flatten).to match_array([1, 2])
-      end
-    end
-  end
-
-end

data/spec/canvas_sync/jobs/beta_cleanup/delete_temp_tables_spec.rb

@@ -1,19 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe CanvasSync::Jobs::BetaCleanup::DeleteTempTablesJob do
-  describe '#perform' do
-    let(:conn) { ActiveRecord::Base.connection }
-    let(:models_to_sync) { %w[accounts courses]}
-
-    before do
-      CanvasSync::Jobs::BetaCleanup::CreateTempTablesJob.new.perform({ models: models_to_sync })
-    end
-
-    it 'deletes the specified tables' do
-      expect(models_to_sync.all? { |table_name| conn.data_source_exists?("beta_#{table_name}") }).to be true
-      described_class.new.perform({ models: models_to_sync })
-      expect(models_to_sync.all? { |table_name| conn.data_source_exists?("beta_#{table_name}") }).to be false
-      expect(models_to_sync.all? { |table_name| conn.data_source_exists?(table_name) }).to be true
-    end
-  end
-end