canvas_sync 0.26.1 → 0.27.1.beta1

data/lib/canvas_sync/jobs/report_sync_task.rb

@@ -0,0 +1,263 @@
+require "csv"
+
+module CanvasSync
+  module Jobs
+    class ReportSyncTask
+      REPORT_TIMEOUT = 24.hours
+      COMPILATION_TIMEOUT = 3.hours
+      MAX_TRIES = 3
+
+      class FatalReportError < ::RuntimeError; end
+
+      # Hash passed specifically to this report; read-only
+      attr_reader :options
+      # Batch context _before_ initializing the report batch; read-only
+      attr_reader :context
+      # Escape hatch to store additional state on the report batch
+      attr_reader :state
+
+      def initialize(options, context, state)
+        @options = options
+        @context = context
+        @state = state
+      end
+
+      delegate :merge_report_params, to: :class
+
+      class << self
+        def from_context
+          report_batch = JobBatches::Batch.current
+          base_context = report_batch.parent.context
+          state = report_batch.context
+          cls = state[:report_class].constantize
+          cls.new(state[:report_options], base_context, state)
+        end
+
+        def perform_later(options)
+          _with_batch(options) do |batch|
+            StarterJob.perform_later
+          end
+        end
+
+        def perform_now(options)
+          _with_batch(options) do |batch|
+            StarterJob.perform_now
+          end
+        end
+
+        def merge_report_params(options, params={}, term_scope: true)
+          batch_context = JobBatches::Batch.current_context
+          term_scope = options[:canvas_term_id] || batch_context[:canvas_term_id] if term_scope == true
+          if term_scope.present?
+            params[:enrollment_term_id] = term_scope
+          end
+          if (updated_after = batch_context[:updated_after]).present?
+            params[:updated_after] = updated_after
+          end
+          params.merge!(options[:report_params]) if options[:report_params].present?
+          params.merge!(options[:report_parameters]) if options[:report_parameters].present?
+          { parameters: params }
+        end
+
+        def report_name(n)
+          define_method(:report_name) { n }
+        end
+
+        private
+
+        def _with_batch(options, &blk)
+          batch = JobBatches::Batch.new
+          batch.description = "CanvasSync RunReport[#{self.name}] Fiber"
+          batch.allow_context_changes = true
+          batch.context[:report_class] = self.to_s
+          batch.context[:report_options] = options
+          batch.jobs do
+            blk.call(batch)
+          end
+        end
+      end
+
+      def report_name
+        raise NotImplementedError
+      end
+
+      def report_parameters
+        merge_report_params(options)
+      end
+
+      def process(file)
+        if (m = self.class.name.match(/Sync(\w+)Job$/)) && m[1].present?
+          model_name = m[1].singularize
+          model = model_name.safe_constantize
+          return do_bulk_import(file, model) if model.present?
+        end
+        raise NotImplementedError
+      end
+
+      def check_frequency
+        Rails.env.development? ? 5.seconds : 1.minute
+      end
+
+      def max_tries
+        3
+      end
+
+      def caching_key
+        Digest::MD5.hexdigest(report_parameters.to_s)[0..8]
+      end
+
+      protected
+
+      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil, &blk)
+        Processors::ReportProcessor.new().do_bulk_import(report_file_path, model, options: options, mapping_key: mapping_key, &blk)
+      end
+
+      class ReportTaskJob < CanvasSync::Job
+        protected
+
+        def canvas_account_id
+          report_task.options[:account_id] || batch_context[:account_id] || "self"
+        end
+
+        def trigger_canvas_report
+          report_name = report_task.report_name
+          report_params = report_task.report_parameters
+
+          caching_config = report_task.options[:caching] || batch_context[:report_caching] || nil
+          cache_key = caching_config && report_task.caching_key
+
+          if cache_key.present?
+            # Be default, only cache reports w/i the same SyncBatch
+            cache_scope = caching_config[:scope]&.to_sym || :sync_batch
+            cache_key = "SyncBatch[#{batch_context[:sync_batch_id]}]:#{cache_key}" if cache_scope == :sync_batch
+
+            batch_context[:caching_key] = cache_key
+
+            cached_report = CanvasSync.redis.hgetall(cache_key)
+            if cached_report.present? && cached_report["started_at"] > (caching_config[:max_age] || 23.hours).ago.iso8601
+              batch_context[:report_id] = cached_report["report_id"]
+              batch_context[:sync_start_time] = cached_report["started_at"]
+              Rails.logger.info("Using cached report execution for #{report_name} with cache key #{cache_key}")
+              return
+            end
+          end
+
+          report = CanvasSync.get_canvas_sync_client(batch_context).start_report(canvas_account_id, report_name, report_params)
+          batch_context[:report_id] = "#{report_name}/#{report['id']}"
+          batch_context[:sync_start_time] = DateTime.now.utc.iso8601 # "report_start_time" would be a better name, but backwards compat
+          batch_context.delete(:compiling_since)
+
+          if cache_key.present?
+            CanvasSync.redis do |r|
+              r.hset(cache_key, {
+                report_id: batch_context[:report_id],
+                started_at: batch_context[:sync_start_time],
+              })
+              r.expire(cache_key, (caching_config[:max_age] || 23.hours).to_i)
+            end
+          end
+
+          report
+        end
+
+        def report_task
+          @report_task ||= ReportSyncTask.from_context
+        end
+
+        def enqueue_checker!
+          report_task.class::CheckerJob.set(wait: report_task.check_frequency).perform_later()
+        end
+      end
+
+      class StarterJob < ReportTaskJob
+        def perform()
+          batch_context[:report_attempt] = 1
+          trigger_canvas_report
+          enqueue_checker!
+        end
+      end
+
+      class CheckerJob < ReportTaskJob
+        def perform()
+          options = report_task.options
+          report_status = CanvasSync.get_canvas_sync_client(batch_context).report_status(canvas_account_id, *batch_context[:report_id].split("/"))
+
+          case report_status["status"].downcase
+          when "complete"
+            ProcessJob.perform_later(report_status["attachment"]["url"])
+          when "error", "deleted"
+            max_tries = options[:report_max_tries] || batch_context[:report_max_tries] || report_task.max_tries || MAX_TRIES
+            report_attempt = batch_context[:report_attempt]
+
+            message = "Report failed to process; status was #{report_status} for #{batch_context[:report_id]}. This report has now failed #{batch_context[:report_attempt]} times."
+            Rails.logger.error(message)
+
+            if report_attempt >= max_tries
+              Rails.logger.error("This report has failed #{report_attempt} times. Giving up.")
+              raise FatalReportError, message
+            else
+              batch_context[:report_attempt] += 1
+
+              # Remove the cache entry (if it wasn't already restarted by some other cacher) so that we actually start a new report
+              if batch_context[:caching_key].present?
+                CanvasSync.redis do |r|
+                  # This could _technically_ result in a race condition, but the chances are quite low and the
+                  #   consequences are just that an extra report is triggered (which Canvas may de-dup anyway)
+                  centry = r.hgetall(batch_context[:caching_key])
+                  if centry.present? && centry["report_id"] == batch_context[:report_id]
+                    r.del(batch_context[:caching_key])
+                    Rails.logger.info("Deleted cache entry for #{batch_context[:report_id]} at #{batch_context[:caching_key]} due to report failure")
+                  end
+                end
+              end
+
+              trigger_canvas_report
+              enqueue_checker!
+            end
+          else
+            report_timeout = options[:report_timeout] || batch_context[:report_timeout] || REPORT_TIMEOUT
+            if timeout_met?(batch_context[:sync_start_time], report_timeout)
+              raise FatalReportError, "Report appears to be stuck #{batch_context[:report_id]}"
+            end
+
+            if report_status["status"].downcase == 'compiling'
+              batch_context[:compiling_since] ||= DateTime.now.iso8601
+              compilation_timeout = options[:report_compilation_timeout] || batch_context[:report_compilation_timeout] || COMPILATION_TIMEOUT
+              if timeout_met?(batch_context[:compiling_since], compilation_timeout)
+                raise FatalReportError, "Report appears to be stuck #{batch_context[:report_id]}"
+              end
+            end
+
+            enqueue_checker!
+          end
+        end
+
+        protected
+
+        def timeout_met?(base_time, timeout_length)
+          return false unless base_time.present? && timeout_length.present?
+          DateTime.now > (DateTime.parse(base_time) + timeout_length)
+        end
+      end
+
+      class ProcessJob < ReportTaskJob
+        def perform(report_url)
+          report_name = batch_context[:report_id].gsub(/\//, "_")
+          download(report_name, report_url) do |file_path|
+            report_task.process(file_path)
+          end
+        end
+
+        private
+
+        def download(report_name, report_url)
+          Dir.mktmpdir do |dir|
+            file_path = "#{dir}/#{report_name}"
+            canvas_sync_client.download_report(report_url, file_path)
+            yield file_path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_accounts_job.rb

@@ -1,6 +1,6 @@
 module CanvasSync
   module Jobs
-    class SyncAccountsJob < ReportStarter
+    class SyncAccountsJob < CanvasSync::Job
       # Starts a provisioning report for just accounts.
       #
       # Provisioning reports do not scope accounts by term, so when we are
@@ -14,15 +14,18 @@ module CanvasSync
           update_or_create_model(Account, acc_params)
         end
 
-        super(
-          "proservices_provisioning_csv",
+        ReportTask.perform_later(options)
+      end
+
+      class ReportTask < ReportSyncTask
+        report_name "proservices_provisioning_csv"
+
+        def report_parameters
           merge_report_params(options, {
             accounts: true,
             include_deleted: true,
-          }, term_scope: false),
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          { models: ["accounts"] },
-        )
+          }, term_scope: false)
+        end
       end
     end
   end

data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb

@@ -1,20 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncAssignmentGroupsJob < ReportStarter
-      # Syncs AssignmentGroups
-      #
-      # Starts a report processor for the assignment_groups report
-      # (the proserv_assignment_group_export_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_assignment_group_export_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::AssignmentGroupsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncAssignmentGroupsJob < ReportSyncTask
+      report_name "proserv_assignment_group_export_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_assignment_overrides_job.rb

@@ -1,19 +1,31 @@
 module CanvasSync
   module Jobs
-    class SyncAssignmentOverridesJob < ReportStarter
-      # Syncs SyncAssignmentOverrides
-      #
-      # Starts a report processor for the assignment overrides report
-      # (the proserv_assignment_overrides_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_assignment_overrides_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::AssignmentOverridesProcessor.to_s,
-          {},
-        )
+    class SyncAssignmentOverridesJob < ReportSyncTask
+      report_name "proserv_assignment_overrides_csv"
+
+      def process(report_file_path)
+        do_bulk_import(report_file_path, AssignmentOverride) do |row|
+          # Handle transforms here instead of in mappings
+          row[:student_ids]       = parse_student_ids(row[:student_ids])
+
+          # Convert empty/null/'0' values to nil
+          # For Ruby 2.7
+          row[:group_id]          = parse_nil_value(row[:group_id])
+          row[:course_section_id] = parse_nil_value(row[:course_section_id])
+          row
+        end
+      end
+
+      private
+
+      def parse_student_ids(value)
+        value.present? ? JSON.parse(value) : nil
+      end
+
+      def parse_nil_value(value)
+        # Convert "0", "null", or empty string to nil
+        return nil if %w[0 null].include?(value) || value == ''
+        value
       end
     end
   end

data/lib/canvas_sync/jobs/sync_assignments_job.rb

@@ -1,20 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncAssignmentsJob < ReportStarter
-      # Syncs Assignments
-      #
-      # Starts a report processor for the assignment report
-      # (the proserv_assignment_export_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_assignment_export_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::AssignmentsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncAssignmentsJob < ReportSyncTask
+      report_name "proserv_assignment_export_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_content_migrations_job.rb

@@ -1,20 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncContentMigrationsJob < ReportStarter
-      # Syncs ContentMigrations
-      #
-      # Starts a report processor for the content migrations report
-      # (the proserv_content_migrations_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_content_migrations_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::ContentMigrationsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncContentMigrationsJob < ReportSyncTask
+      report_name "proserv_content_migrations_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_context_module_items_job.rb

@@ -1,20 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncContextModuleItemsJob < ReportStarter
-      # Syncs ContextModuleItems
-      #
-      # Starts a report processor for the context modules report
-      # (the proserv_context_module_items_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_context_module_items_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::ContextModuleItemsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncContextModuleItemsJob < ReportSyncTask
+      report_name "proserv_context_module_items_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_context_modules_job.rb

@@ -1,20 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncContextModulesJob < ReportStarter
-      # Syncs ContextModules
-      #
-      # Starts a report processor for the context modules report
-      # (the proserv_context_modules_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "proserv_context_modules_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::ContextModulesProcessor.to_s,
-          {},
-        )
-      end
+    class SyncContextModulesJob < ReportSyncTask
+      report_name "proserv_context_modules_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_course_progresses_job.rb

@@ -1,21 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncCourseProgressesJob < ReportStarter
-      # Syncs CourseProgresses
-      #
-      # Starts a report processor for the course completion report
-      # (the proserv_course_completion_csv report must be enabled)
-      #
-      # @param options [Hash]
-      def perform(options)
-        super(
-          'proserv_course_completion_csv',
-          merge_report_params(options),
-          CanvasSync::Processors::CourseCompletionReportProcessor.to_s,
-          {},
-        )
-      end
+    class SyncCourseProgressesJob < ReportSyncTask
+      report_name "proserv_course_completion_csv"
     end
   end
-
 end

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -1,20 +1,22 @@
+require "zip"
+require "zip/version"
+
 module CanvasSync
   module Jobs
-    # ActiveJob class that starts a Canvas provisioning report
-    class SyncProvisioningReportJob < ReportStarter
-      def perform(options)
-        params = {
-          include_deleted: true,
-        }
+    # Job that syncs a Canvas provisioning report
+    class SyncProvisioningReportJob < ReportSyncTask
+      report_name "proservices_provisioning_csv"
+
+      def report_parameters
+        params = { include_deleted: true }
 
         options[:models].each do |model|
           # group_membership is the only model param that is singular :(
           model = 'group_membership' if model == 'group_memberships'
-
           params[model] = true
         end
 
-        merged_params = merge_report_params(options, params, {}).with_indifferent_access
+        merged_params = merge_report_params(options, params)
 
         # Make sure the report also checks last_activity_at when checking updated_at
         if options[:models].include?("enrollments")
@@ -23,12 +25,131 @@ module CanvasSync
           end
         end
 
-        super(
-          "proservices_provisioning_csv",
-          merged_params,
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          options,
-        )
+        merged_params
+      end
+
+      def process(file)
+        if options[:models].length == 1
+          run_import(options[:models][0], file)
+        else
+          unzipped_file_dir = extract(file)
+          Dir[unzipped_file_dir + "/*.csv"].sort.each do |file_path|
+            model_name = file_path.split("/").last.split(".").first
+            run_import(model_name, file_path)
+          end
+        end
+      end
+
+      private
+
+      def extract(file_path)
+        unzipped_file_dir = "#{file_path}_unzipped"
+
+        Zip::File.open(file_path) do |zip_file|
+          zip_file.each do |f|
+            f_path = File.join(unzipped_file_dir, f.name)
+            FileUtils.mkdir_p(File.dirname(f_path))
+            next if File.exist?(f_path)
+
+            if Zip::VERSION >= "3.0.0"
+              zip_file.extract(f, destination_directory: unzipped_file_dir)
+            else
+              zip_file.extract(f, f_path)
+            end
+          end
+        end
+
+        unzipped_file_dir
+      end
+
+      def run_import(model_name, report_file_path)
+        if legacy?(model_name)
+          CanvasSync::Importers::LegacyImporter.import(
+            report_file_path,
+            model_name.singularize.capitalize.constantize,
+            options[:account_id],
+            options,
+          )
+        else
+          send("bulk_process_#{model_name}", report_file_path)
+        end
+      end
+
+      def legacy?(model_name)
+        opt = options[:legacy_support]
+        return false if opt == false || opt.nil?
+        return true if opt == true
+        return opt.include?(model_name)
+      end
+
+      def bulk_process_users(report_file_path)
+        do_bulk_import(report_file_path, User)
+      end
+
+      def bulk_process_user_observers(report_file_path)
+        do_bulk_import(report_file_path, UserObserver)
+      end
+
+      def bulk_process_pseudonyms(report_file_path)
+        do_bulk_import(report_file_path, Pseudonym)
+      end
+
+      def bulk_process_accounts(report_file_path)
+        do_bulk_import(report_file_path, Account)
+      end
+
+      def bulk_process_courses(report_file_path)
+        do_bulk_import(report_file_path, Course)
+      end
+
+      def bulk_process_enrollments(report_file_path)
+        do_bulk_import(report_file_path, Enrollment)
+      end
+
+      def bulk_process_sections(report_file_path)
+        do_bulk_import(report_file_path, Section)
+      end
+
+      def bulk_process_xlist(report_file_path)
+        do_bulk_import(report_file_path, Section, mapping_key: :xlist)
+      end
+
+      def bulk_process_groups(report_file_path)
+        do_bulk_import(report_file_path, Group)
+      end
+
+      def bulk_process_grading_periods(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriod)
+      end
+
+      def bulk_process_grading_period_groups(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriodGroup)
+      end
+
+      # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
+      def bulk_process_group_membership(report_file_path)
+        do_bulk_import(report_file_path, GroupMembership)
+      end
+
+      def bulk_process_learning_outcomes(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcome) do |row|
+          row[:root_account_ids] = JSON.parse row[:root_account_ids]
+          row
+        end
+      end
+
+      def bulk_process_learning_outcome_results(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcomeResult)
+      end
+
+      def bulk_process_course_nicknames(report_file_path)
+        exists = []
+        do_bulk_import(report_file_path, CourseNickname) do |row|
+          exists << row[:user_preference_value_id]
+          row
+        end
+        # Canvas does not soft delete UserPreferenceValues so there's no point in doing the same
+        CourseNickname.where.not(canvas_user_preference_value_id: exists).or(CourseNickname.where(canvas_user_preference_value_id: nil)).delete_all
       end
     end
   end

data/lib/canvas_sync/jobs/sync_rubric_assessments_job.rb

@@ -1,15 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncRubricAssessmentsJob < ReportStarter
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "rubric_assessments_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::RubricAssessmentsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncRubricAssessmentsJob < ReportSyncTask
+      report_name "rubric_assessments_csv"
     end
   end
 end

data/lib/canvas_sync/jobs/sync_rubric_associations_job.rb

@@ -1,15 +1,7 @@
 module CanvasSync
   module Jobs
-    class SyncRubricAssociationsJob < ReportStarter
-      # @param options [Hash]
-      def perform(options)
-        super(
-          "rubric_associations_csv",
-          merge_report_params(options),
-          CanvasSync::Processors::RubricAssociationsProcessor.to_s,
-          {},
-        )
-      end
+    class SyncRubricAssociationsJob < ReportSyncTask
+      report_name "rubric_associations_csv"
     end
   end
 end