canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/jobs/sync_rubrics_job.rb

@@ -0,0 +1,15 @@
+module CanvasSync
+  module Jobs
+    class SyncRubricsJob < ReportStarter
+      # @param options [Hash]
+      def perform(options)
+        super(
+          "rubrics_csv",
+          merge_report_params(options),
+          CanvasSync::Processors::RubricsProcessor.to_s,
+          {},
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/sync_simple_table_job.rb

@@ -5,36 +5,19 @@ module CanvasSync
       # Starts a report processor for the specified report
       # (the specified report must be enabled)
       #
-      # @param job_chain [Hash]
-      # @param options [Hash]
-      def perform(job_chain, options)
-        if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
-            Term.send(options[:term_scope]).find_each.map do |term|
-              # Deep copy the job_chain so each report gets the correct term id passed into
-              # its options with no side effects
-              term_id = get_term_id(term)
-              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-              duped_job_chain[:global_options][:canvas_term_id] = term_id
-              {
-                job_chain: duped_job_chain,
-                params: report_params(options, term_id),
-                options: options,
-              }
-            end
-          end
-
-          sub_reports.each do |r|
-            start_report(r[:params], r[:job_chain], r[:options])
-          end
-        else
-          start_report(report_params(options), job_chain, options)
-        end
+      # @param options [Hash] {
+      #   report_name: "", # Name of the report in the Canvas API
+      #   model: , # Model to map the report results as
+      #   mapping: {} # Mapping to describe how to convert the report to the Model,
+      #   klass: ,
+      #   params: {}, # List of params to pass to the report
+      # }
+      def perform(options)
+        start_report(report_params(options), options)
       end
 
-      def start_report(params, job_chain, options)
+      def start_report(params, options)
         CanvasSync::Jobs::ReportStarter.perform_later(
-          job_chain,
           options[:report_name],
           params,
           CanvasSync::Processors::NormalProcessor.to_s,
@@ -42,15 +25,11 @@ module CanvasSync
         )
       end
 
-      def report_params(options, canvas_term_id=nil)
+      def report_params(options, canvas_term_id = options[:canvas_term_id] || batch_context[:canvas_term_id])
         params = options[:params] || {}
         params["parameters[enrollment_term_id]"] = canvas_term_id if canvas_term_id
         params
       end
-
-      def get_term_id(term)
-        term.try(:canvas_id) || term.canvas_term_id
-      end
     end
   end
 end

data/lib/canvas_sync/jobs/sync_submissions_job.rb

@@ -6,13 +6,15 @@ module CanvasSync
       # Starts a report processor for the submission report
       # (the proserv_student_submissions_csv report must be enabled)
       #
-      # @param job_chain [Hash]
       # @param options [Hash]
-      def perform(job_chain, options)
+      def perform(options)
         super(
-          job_chain,
           "proserv_student_submissions_csv",
-          merge_report_params(job_chain, options, { include_all: options[:include_all] }, {}),
+          merge_report_params(options, {
+            include_all: options[:include_all],
+            enrollment_batch_size: options[:enrollment_batch_size],
+            include_all_except: options[:include_all_except]
+          }, {}),
           CanvasSync::Processors::SubmissionsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -5,16 +5,17 @@ module CanvasSync
       #
       # Terms are pre-synced so that provisioning reports can be scoped to term.
       #
-      # @param job_chain [Hash]
-      # @param options [Hash]
-      def perform(job_chain, _options)
-        CanvasSync.get_canvas_sync_client(job_chain[:global_options]).terms("self").all_pages!.each do |term_params|
-          if job_chain[:global_options][:account_id]
+      # @param options [Hash] If options contains a :term_scope a seperate provisioning report
+      #   will be started for each term in that scope. :models should be an array of
+      #   models to sync.
+      def perform(options)
+        CanvasSync.get_canvas_sync_client(batch_context).terms("self", workflow_state: ['all']).all_pages!.each do |term_params|
+          if account_id = batch_context[:account_id]
             # These branches are primarily to support Legacy apps
             if Term.respond_to?(:create_or_update) && Term.method(:create_or_update).arity.abs == 2
-              Term.create_or_update(term_params, job_chain[:global_options][:account_id])
+              Term.create_or_update(term_params, account_id)
             else
-              term_params[:account_id] |= job_chain[:global_options][:account_id]
+              term_params[:account_id] |= account_id
               update_or_create_model(Term, term_params)
             end
           else
@@ -22,7 +23,7 @@ module CanvasSync
           end
         end
 
-        CanvasSync.invoke_next(job_chain)
+        TermBatchesJob.perform_now(options)
       end
     end
   end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -0,0 +1,50 @@
+module CanvasSync
+  module Jobs
+    class TermBatchesJob < CanvasSync::Job
+      def perform(options)
+        if (jobs = options[:sub_jobs]).present?
+          context = options[:context] || {}
+          if options[:term_scope]
+            Term.send(options[:term_scope]).find_each.map do |term|
+              term_id = get_term_id(term)
+              local_context = context.merge(canvas_term_id: term_id)
+
+              # Override the delta-syncing date if:
+              # 1. the Term hasn't been synced before or
+              # 2. the Term underwent a period of not syncing
+              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
+              if batch_context[:updated_after]
+                if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
+                  local_context[:updated_after] = term_last_sync.presence
+                end
+              end
+
+              JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
+                b.description = "TermBatchJob(#{term_id}) Root"
+                b.context = local_context
+                b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
+              end
+            end
+          else
+            JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
+          end
+        end
+      end
+
+      def self.batch_finished(status, opts)
+        ctx = JobBatches::Batch.current_context
+        term_id = ctx[:canvas_term_id]
+        CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
+      end
+
+      def self.last_sync_key(term_id)
+        ctx = JobBatches::Batch.current_context
+        "#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
+      end
+
+      def get_term_id(term)
+        term.try(:canvas_id) || term.canvas_term_id
+      end
+    end
+  end
+end

data/lib/canvas_sync/{generators/templates/services/live_events/base_event.rb → live_events/base_handler.rb}

@@ -1,29 +1,25 @@
-# <%= autogenerated_event_warning %>
-
-module LiveEvents
+module CanvasSync::LiveEvents
   class RetryLiveEventJob < StandardError; end
 
-  class BaseEvent < CanvasSync::Job
+  # Deprecated
+  class BaseHandler < CanvasSync::Job
     attr_accessor :raw_payload
     attr_accessor :payload
     attr_accessor :metadata
 
     def perform(event_payload)
       @raw_payload = event_payload
-      @metadata = HashWithIndifferentAccess.new(event_payload["attributes"])
+      @metadata = HashWithIndifferentAccess.new(event_payload["metadata"])
       @payload = HashWithIndifferentAccess.new(event_payload["body"])
       process_with_retry
     end
 
+    protected
+
     def process
       raise "process must be implemented in your subclass"
     end
 
-    # Live events will use a canvas global ID (cross shard) for any ID's provided. This method will return the local ID.
-    def local_canvas_id(id)
-      id.to_i % 10_000_000_000_000
-    end
-
     private
 
     # Sometimes a creation and update event get sent by Canvas at almost the exact same time. This means

data/lib/canvas_sync/live_events/process_event_job.rb

@@ -0,0 +1,26 @@
+
+module CanvasSync::LiveEvents
+  class ProcessEventJob < ::CanvasSync::Job
+    def perform(event)
+      event = event.with_indifferent_access
+
+      # Support Legacy-Style Event Handlers
+      begin
+        legacy_handler = "LiveEvents::#{event['metadata']['event_name'].camelcase}Event".constantize
+        legacy_handler.perform_later(event)
+        return
+      rescue LoadError, NameError => e
+        Rails.logger.error("Error accessing legacy-style LiveEvents handler #{e}, #{event}")
+      end
+
+      handlers = CanvasSync::LiveEvents.registered_handlers
+      handlers.each do |h|
+        h.call(event)
+      rescue => e
+        throw e if Rails.env.test?
+        Rails.logger.error("LiveEvent Handler raised Error: #{e}")
+        Raven.captureException(e) if defined? Raven
+      end
+    end
+  end
+end

data/lib/canvas_sync/live_events.rb

@@ -0,0 +1,38 @@
+require_relative './live_events/base_handler.rb'
+require_relative './live_events/process_event_job.rb'
+
+module CanvasSync
+  module LiveEvents
+
+    @@registered_handlers = []
+
+    class << self
+      def listen(event_types = nil, &blk)
+        if event_types != nil
+          blk = wrap_method(blk) do |inner, event|
+            meta = event[:metadata]
+            payload = event[:payload]
+
+            if event_types.include?(meta[:event_name])
+              inner.call(*args)
+            end
+          end
+        end
+
+        @@registered_handlers << blk
+      end
+
+      def registered_handlers
+        @@registered_handlers
+      end
+
+      private
+
+      def wrap_method(inner, &outer)
+        ->(*args) {
+          outer.call(inner, *args)
+        }
+      end
+    end
+  end
+end

data/lib/canvas_sync/misc_helper.rb

@@ -0,0 +1,63 @@
+require 'active_record'
+
+module CanvasSync
+  module MiscHelper
+    MigrationClass = Rails.version < '5.0' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+
+    def self.to_boolean(v)
+      if Rails.version < '5.0'
+        ActiveRecord::Type::Boolean.new.type_cast_from_user(v)
+      else
+        ActiveRecord::Type::Boolean.new.deserialize(v)
+      end
+    end
+
+    def self.invoke_task(job)
+      job = job.symbolize_keys
+
+      job_args = job[:args] || job[:parameters] || []
+      job_kwargs = job[:kwargs] || {}
+
+      if (mthd = job[:method]) && !(job[:class] || job[:instance_of] || job[:model])
+        if mthd.include?('#')
+          clazz, method = clazz.split("#")
+          job[:instance_of] = clazz
+          job[:method] = method
+        elsif mthd.include?('.')
+          clazz, method = mthd.split(".")
+          job[:class] = clazz
+          job[:method] = method
+        end
+      end
+
+      if job[:model]
+        # TODO Support globalid
+        model_class = load_constant(job[:model])
+        find_by = job[:find_by]
+        target = find_by.is_a?(Hash) ? model_class.find_by(find_by) : model_class.find_by(id: find_by)
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:class]
+        target = load_constant(job[:class])
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:instance_of]
+        target = load_constant(job[:instance_of]).new
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:job]
+        job_class = load_constant(job[:job])
+        job_class = job_class.set(job[:options]) if job[:options].present?
+        if job_class < ActiveJob::Base
+          job_class.perform_later(*job_args, **job_kwargs)
+        else
+          job_args << job_kwargs.symbolize_keys if job_kwargs
+          # job_args[-1] = job_args[-1].symbolize_keys if job_args[-1].is_a?(Hash)
+          job_class.perform_async(*job_args)
+        end
+      end
+    end
+
+    def self.load_constant(const)
+      const = const.constantize if const.is_a?(String)
+      const
+    end
+  end
+end

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,16 +8,11 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignment_groups][:report_columns],
-          AssignmentGroup,
-          mapping[:assignment_groups][:conflict_target].to_sym,
-        )
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, AssignmentGroup, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,16 +8,11 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignments][:report_columns],
-          Assignment,
-          mapping[:assignments][:conflict_target].to_sym,
-        )
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, Assignment, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/content_migrations_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a content migrations report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class ContentMigrationsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, ContentMigration, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,16 +8,11 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_module_items][:report_columns],
-          ContextModuleItem,
-          mapping[:context_module_items][:conflict_target].to_sym,
-        )
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, ContextModuleItem, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,16 +8,11 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_modules][:report_columns],
-          ContextModule,
-          mapping[:context_modules][:conflict_target].to_sym,
-        )
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, ContextModule, options: options)
       end
     end
   end