canvas_sync 0.16.5 → 0.17.0.beta1

data/lib/canvas_sync/jobs/report_starter.rb

@@ -2,7 +2,6 @@ module CanvasSync
   module Jobs
     # Starts a Canvas report and enqueues a ReportChecker
     class ReportStarter < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_params [Hash] The Canvas report parameters
       # @param processor [String] a stringified report processor class name
@@ -10,31 +9,39 @@ module CanvasSync
       # @param allow_redownloads [Boolean] whether you want the job_chain to cache this report,
       #   so that any later jobs in the chain will use the same generated report
       # @return [nil]
-      def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
-        account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-	options[:sync_start_time] = DateTime.now.utc.iso8601
-        report_id = if allow_redownloads
-                      get_cached_report(job_chain, account_id, report_name, report_params)
-                    else
-                      start_report(job_chain, account_id, report_name, report_params)
-                    end
+      def perform(report_name, report_params, processor, options, allow_redownloads: false)
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
 
-        CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
-          job_chain,
-          report_name,
-          report_id,
-          processor,
-          options,
-        )
+        report_id = start_report(account_id, report_name, report_params)
+        # TODO: Restore report caching support (does nayone actually use it?)
+        # report_id = if allow_redownloads
+        #               get_cached_report(account_id, report_name, report_params)
+        #             else
+        #               start_report(account_id, report_name, report_params)
+        #             end
+
+        batch = JobBatches::Batch.new
+        batch.description = "CanvasSync #{report_name} Fiber"
+        batch.jobs do
+          CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
+            report_name,
+            report_id,
+            processor,
+            options,
+          )
+        end
       end
 
       protected
 
-      def merge_report_params(job_chain, options={}, params={}, term_scope: true)
-        term_scope = job_chain[:global_options][:canvas_term_id] if term_scope == true
+      def merge_report_params(options={}, params={}, term_scope: true)
+        term_scope = batch_context[:canvas_term_id] if term_scope == true
         if term_scope.present?
           params[:enrollment_term_id] = term_scope
         end
+        if (updated_after = batch_context[:updated_after]).present?
+          params[:updated_after] = updated_after
+        end
         params.merge!(options[:report_params]) if options[:report_params].present?
         { parameters: params }
       end
@@ -42,6 +49,7 @@ module CanvasSync
       private
 
       def get_cached_report(job_chain, account_id, report_name, report_params)
+        # TODO: job_chain[:global_options] is no longer available and batch_context won't work for this
         if job_chain[:global_options][report_name].present?
           job_chain[:global_options][report_name]
         else
@@ -51,8 +59,8 @@ module CanvasSync
         end
       end
 
-      def start_report(job_chain, account_id, report_name, report_params)
-        report = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+      def start_report(account_id, report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(batch_context)
                            .start_report(account_id, report_name, report_params)
         report["id"]
       end

data/lib/canvas_sync/jobs/sync_accounts_job.rb

@@ -7,18 +7,16 @@ module CanvasSync
       # running provisioning by term we sync users first so we don't duplicate
       # the work of syncing all accounts for each term.
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         unless options[:root_account] == false
-          acc_params = CanvasSync.get_canvas_sync_client(job_chain[:global_options]).account("self")
+          acc_params = CanvasSync.get_canvas_sync_client(batch_context).account("self")
           update_or_create_model(Account, acc_params)
         end
 
         super(
-          job_chain,
           "proservices_provisioning_csv",
-          merge_report_params(job_chain, options, {
+          merge_report_params(options, {
             accounts: true,
             include_deleted: true,
           }, term_scope: false),

data/lib/canvas_sync/jobs/sync_admins_job.rb

@@ -4,11 +4,10 @@ module CanvasSync
       # Syncs Admins using the Canvas API
       #
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, _options)
+      def perform(options)
         updated_admin_ids = []
-        api_client = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+        api_client = CanvasSync.get_canvas_sync_client(batch_context)
         CanvasSync.sync_scope(Account).find_each do |acc|
           api_client.account_admins(acc.canvas_id).all_pages_each do |admin_params|
             admin_params[:account_id] = acc.canvas_id
@@ -17,7 +16,6 @@ module CanvasSync
           end
         end
         Admin.where.not(id: updated_admin_ids).update_all(workflow_state: 'inactive')
-        CanvasSync.invoke_next(job_chain)
       end
     end
   end

data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb

@@ -6,13 +6,11 @@ module CanvasSync
       # Starts a report processor for the assignment_groups report
       # (the proserv_assignment_group_export_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_assignment_group_export_csv",
-          merge_report_params(job_chain, options),
+          merge_report_params(options),
           CanvasSync::Processors::AssignmentGroupsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_assignments_job.rb

@@ -6,13 +6,11 @@ module CanvasSync
       # Starts a report processor for the assignment report
       # (the proserv_assignment_export_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_assignment_export_csv",
-          merge_report_params(job_chain, options),
+          merge_report_params(options),
           CanvasSync::Processors::AssignmentsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_context_module_items_job.rb

@@ -6,13 +6,11 @@ module CanvasSync
       # Starts a report processor for the context modules report
       # (the proserv_context_module_items_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_context_module_items_csv",
-          merge_report_params(job_chain, options),
+          merge_report_params(options),
           CanvasSync::Processors::ContextModuleItemsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_context_modules_job.rb

@@ -6,13 +6,11 @@ module CanvasSync
       # Starts a report processor for the context modules report
       # (the proserv_context_modules_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_context_modules_csv",
-          merge_report_params(job_chain, options),
+          merge_report_params(options),
           CanvasSync::Processors::ContextModulesProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -2,40 +2,17 @@ module CanvasSync
   module Jobs
     # ActiveJob class that starts a Canvas provisioning report
     class SyncProvisioningReportJob < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param options [Hash] If options contains a :term_scope a seperate provisioning report
       #   will be started for each term in that scope. :models should be an array of
       #   models to sync.
-      def perform(job_chain, options)
-        if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
-            Term.send(options[:term_scope]).find_each.map do |term|
-              fork = fork_template.duplicate
-              # Deep copy the job_chain so each report gets the correct term id passed into
-              # its options with no side effects
-              term_id = get_term_id(term)
-              fork[:global_options][:canvas_term_id] = term_id
-              {
-                job_chain: fork.serialize,
-                params: report_params(options, term_id),
-                options: options,
-              }
-            end
-          end
-
-          sub_reports.each do |r|
-            start_report(r[:params], r[:job_chain], r[:options])
-          end
-        else
-          start_report(report_params(options), job_chain, options)
-        end
+      def perform(options)
+        start_report(report_params(options), options)
       end
 
       protected
 
-      def start_report(report_params, job_chain, options)
+      def start_report(report_params, options)
         CanvasSync::Jobs::ReportStarter.perform_later(
-          job_chain,
           "proservices_provisioning_csv",
           report_params,
           CanvasSync::Processors::ProvisioningReportProcessor.to_s,
@@ -43,7 +20,7 @@ module CanvasSync
         )
       end
 
-      def report_params(options, canvas_term_id=nil)
+      def report_params(options, canvas_term_id = options[:canvas_term_id] || batch_context[:canvas_term_id])
         params = {
           include_deleted: true,
         }
@@ -60,10 +37,6 @@ module CanvasSync
 
         { parameters: params }
       end
-
-      def get_term_id(term)
-        term.try(:canvas_id) || term.canvas_term_id
-      end
     end
   end
 end

data/lib/canvas_sync/jobs/sync_roles_job.rb

@@ -3,12 +3,10 @@ module CanvasSync
     class SyncRolesJob < CanvasSync::Job
       # Syncs Roles using the Canvas API
       #
-      #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, _options)
+      def perform(options)
         updated_role_ids = []
-        api_client = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+        api_client = CanvasSync.get_canvas_sync_client(batch_context)
         CanvasSync.sync_scope(Account).find_each do |acc|
           api_client.list_roles(acc.canvas_id, state: %w[active inactive]).all_pages_each do |role_params|
             role = update_or_create_model(Role, role_params)
@@ -16,7 +14,6 @@ module CanvasSync
           end
         end
         Role.where.not(id: updated_role_ids).update_all(workflow_state: 'inactive')
-        CanvasSync.invoke_next(job_chain)
       end
     end
   end

data/lib/canvas_sync/jobs/sync_simple_table_job.rb

@@ -5,36 +5,19 @@ module CanvasSync
       # Starts a report processor for the specified report
       # (the specified report must be enabled)
       #
-      # @param job_chain [Hash]
-      # @param options [Hash]
-      def perform(job_chain, options)
-        if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
-            Term.send(options[:term_scope]).find_each.map do |term|
-              fork = fork_template.duplicate
-              # Deep copy the job_chain so each report gets the correct term id passed into
-              # its options with no side effects
-              term_id = get_term_id(term)
-              fork[:global_options][:canvas_term_id] = term_id
-              {
-                job_chain: fork.serialize,
-                params: report_params(options, term_id),
-                options: options,
-              }
-            end
-          end
-
-          sub_reports.each do |r|
-            start_report(r[:params], r[:job_chain], r[:options])
-          end
-        else
-          start_report(report_params(options), job_chain, options)
-        end
+      # @param options [Hash] {
+      #   report_name: "", # Name of the report in the Canvas API
+      #   model: , # Model to map the report results as
+      #   mapping: {} # Mapping to describe how to convert the report to the Model,
+      #   klass: ,
+      #   params: {}, # List of params to pass to the report
+      # }
+      def perform(options)
+        start_report(report_params(options), options)
       end
 
-      def start_report(params, job_chain, options)
+      def start_report(params, options)
         CanvasSync::Jobs::ReportStarter.perform_later(
-          job_chain,
           options[:report_name],
           params,
           CanvasSync::Processors::NormalProcessor.to_s,
@@ -42,15 +25,11 @@ module CanvasSync
         )
       end
 
-      def report_params(options, canvas_term_id=nil)
+      def report_params(options, canvas_term_id = options[:canvas_term_id] || batch_context[:canvas_term_id])
         params = options[:params] || {}
         params["parameters[enrollment_term_id]"] = canvas_term_id if canvas_term_id
         params
       end
-
-      def get_term_id(term)
-        term.try(:canvas_id) || term.canvas_term_id
-      end
     end
   end
 end

data/lib/canvas_sync/jobs/sync_submissions_job.rb

@@ -6,13 +6,11 @@ module CanvasSync
       # Starts a report processor for the submission report
       # (the proserv_student_submissions_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_student_submissions_csv",
-          merge_report_params(job_chain, options, { include_all: options[:include_all] }, {}),
+          merge_report_params(options, { include_all: options[:include_all] }, {}),
           CanvasSync::Processors::SubmissionsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -5,16 +5,15 @@ module CanvasSync
       #
       # Terms are pre-synced so that provisioning reports can be scoped to term.
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, _options)
-        CanvasSync.get_canvas_sync_client(job_chain[:global_options]).terms("self").all_pages!.each do |term_params|
-          if job_chain[:global_options][:account_id]
+      def perform(options)
+        CanvasSync.get_canvas_sync_client(batch_context).terms("self").all_pages!.each do |term_params|
+          if account_id = batch_context[:account_id]
             # These branches are primarily to support Legacy apps
             if Term.respond_to?(:create_or_update) && Term.method(:create_or_update).arity.abs == 2
-              Term.create_or_update(term_params, job_chain[:global_options][:account_id])
+              Term.create_or_update(term_params, account_id)
             else
-              term_params[:account_id] |= job_chain[:global_options][:account_id]
+              term_params[:account_id] |= account_id
               update_or_create_model(Term, term_params)
             end
           else
@@ -22,7 +21,23 @@ module CanvasSync
           end
         end
 
-        CanvasSync.invoke_next(job_chain)
+        if (jobs = options[:sub_jobs]).present?
+          context = options[:context] || {}
+          if options[:term_scope]
+            Term.send(options[:term_scope]).find_each.map do |term|
+              local_context = context.merge(canvas_term_id: get_term_id(term))
+              JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
+            end
+          else
+            JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
+          end
+        end
+      end
+
+      protected
+
+      def get_term_id(term)
+        term.try(:canvas_id) || term.canvas_term_id
       end
     end
   end

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,16 +8,15 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path, _options)
+        new(report_file_path)
       end
 
-      def initialize(report_file_path, options)
+      def initialize(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
-          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,16 +8,15 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path, _options)
+        new(report_file_path)
       end
 
-      def initialize(report_file_path, options)
+      def initialize(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
-          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,16 +8,15 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path, _options)
+        new(report_file_path)
       end
 
-      def initialize(report_file_path, options)
+      def initialize(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
-          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,16 +8,15 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path, _options)
+        new(report_file_path)
       end
 
-      def initialize(report_file_path, options)
+      def initialize(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_modules][:report_columns],
           ContextModule,
           mapping[:context_modules][:conflict_target].to_sym,
-          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -18,8 +18,7 @@ module CanvasSync
           report_file_path,
           mapping[options[:mapping].to_sym][:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target,
-          import_args: options
+          conflict_target ? conflict_target.to_sym : conflict_target
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,6 +21,7 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
+
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -74,7 +75,6 @@ module CanvasSync
           mapping[:users][:report_columns],
           User,
           mapping[:users][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -84,7 +84,6 @@ module CanvasSync
           mapping[:pseudonyms][:report_columns],
           Pseudonym,
           mapping[:pseudonyms][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -93,8 +92,7 @@ module CanvasSync
           report_file_path,
           mapping[:accounts][:report_columns],
           Account,
-          mapping[:accounts][:conflict_target].to_sym,
-          import_args: @options
+          mapping[:accounts][:conflict_target].to_sym
         )
       end
 
@@ -104,7 +102,6 @@ module CanvasSync
           mapping[:courses][:report_columns],
           Course,
           mapping[:courses][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -114,7 +111,6 @@ module CanvasSync
           mapping[:enrollments][:report_columns],
           Enrollment,
           mapping[:enrollments][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -124,7 +120,6 @@ module CanvasSync
           mapping[:sections][:report_columns],
           Section,
           mapping[:sections][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -134,7 +129,6 @@ module CanvasSync
           mapping[:xlist][:report_columns],
           Section,
           mapping[:xlist][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -144,7 +138,6 @@ module CanvasSync
           mapping[:groups][:report_columns],
           Group,
           mapping[:groups][:conflict_target].to_sym,
-          import_args: @options
         )
       end
 
@@ -155,7 +148,6 @@ module CanvasSync
           mapping[:group_memberships][:report_columns],
           GroupMembership,
           mapping[:group_memberships][:conflict_target].to_sym,
-          import_args: @options
         )
       end
     end

data/lib/canvas_sync/processors/submissions_processor.rb

@@ -8,16 +8,15 @@ module CanvasSync
     # @param options [Hash]
     class SubmissionsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path, _options)
+        new(report_file_path)
       end
 
-      def initialize(report_file_path, options)
+      def initialize(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:submissions][:report_columns],
           Submission,
           mapping[:submissions][:conflict_target].to_sym,
-          import_args: options
         )
       end
     end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.16.5".freeze
+  VERSION = "0.17.0.beta1".freeze
 end