canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/generators/templates/models/section.rb

@@ -4,6 +4,15 @@ class Section < ApplicationRecord
   include CanvasSync::Record
   include CanvasSync::Concerns::ApiSyncable
 
+  canvas_sync_features :defaults
+
+  # include CanvasSync::Concerns::LiveEventSync
+  # after_process_live_event do
+  #   # A section change could constitute a crosslisting change, which means
+  #   # we need to make sure all our enrollments are pointing to the correct course
+  #   enrollments.update_all(canvas_course_id: canvas_course_id)
+  # end
+
   validates :canvas_id, uniqueness: true, presence: true
   belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_section_id

data/lib/canvas_sync/generators/templates/models/submission.rb

@@ -3,6 +3,9 @@
 class Submission < ApplicationRecord
   include CanvasSync::Record
   include CanvasSync::Concerns::ApiSyncable
+  # include CanvasSync::Concerns::LiveEventSync
+
+  canvas_sync_features :defaults
 
   validates :canvas_id, uniqueness: true, presence: true
   belongs_to :assignment, primary_key: :canvas_id, foreign_key: :canvas_assignment_id, optional: true

data/lib/canvas_sync/generators/templates/models/term.rb

@@ -4,8 +4,11 @@ class Term < ApplicationRecord
   include CanvasSync::Record
   include CanvasSync::Concerns::ApiSyncable
 
+  canvas_sync_features :defaults
+
   validates :canvas_id, uniqueness: true, presence: true
   has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :grading_period_group_id, optional: true
 
   api_syncable({
     canvas_id: :id,

data/lib/canvas_sync/generators/templates/models/user.rb

@@ -4,6 +4,16 @@ class User < ApplicationRecord
   include CanvasSync::Record
   include CanvasSync::Concerns::ApiSyncable
 
+  canvas_sync_features :defaults
+
+  # include CanvasSync::Concerns::LiveEventSync
+  # around_process_live_event do |user, blk|
+  #   blk.call
+  # rescue Footrest::HttpError::Unauthorized => e
+  #   # This can happen when a new user is created, but hasn't setup a login on Canvas yet.
+  #   Rails.logger.info("Failed to fetch user #{canvas_user_id}: #{e.backtrace}")
+  # end
+
   validates :canvas_id, uniqueness: true, presence: true
   has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
@@ -11,6 +21,7 @@ class User < ApplicationRecord
   has_many :admin_roles, through: :admins, source: :role
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_user_id
+  has_many :rubrics, primary_key: :canvas_id, foreign_key: :canvas_user_id
 
   api_syncable({
     sis_id: :sis_user_id,

data/lib/canvas_sync/generators/templates/models/user_observer.rb

@@ -0,0 +1,13 @@
+# <%= autogenerated_model_warning %>
+
+class UserObserver < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  canvas_sync_features :defaults
+
+  validates :canvas_id, uniqueness: true, presence: true
+
+  belongs_to :observing_user, primary_key: :canvas_id, foreign_key: :observing_user_id, class_name: 'User', optional: true
+  belongs_to :observed_user, primary_key: :canvas_id, foreign_key: :observed_user_id, class_name: 'User', optional: true
+end

data/lib/canvas_sync/generators/templates/services/live_events/assignment_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class AssignmentEvent < LiveEvents::BaseEvent
+  class AssignmentEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       canvas_assignment_id = local_canvas_id(payload[:assignment_id])

data/lib/canvas_sync/generators/templates/services/live_events/assignment_group_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class AssignmentGroupEvent < LiveEvents::BaseEvent
+  class AssignmentGroupEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       canvas_assignment_group_id = local_canvas_id(payload[:assignment_group_id])

data/lib/canvas_sync/generators/templates/services/live_events/course_event.rb

@@ -1,8 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class CourseEvent < LiveEvents::BaseEvent
-
+  class CourseEvent < CanvasSync::LiveEvents::BaseHandler
     def process
       course = Course.where(canvas_id: local_canvas_id(payload[:course_id])).first_or_initialize
       course.canvas_account_id = local_canvas_id(payload[:account_id])
@@ -12,7 +11,6 @@ module LiveEvents
       end
       course.sync_from_api
     end
-
   end
 
   class CourseCreatedEvent < LiveEvents::CourseEvent; end

data/lib/canvas_sync/generators/templates/services/live_events/course_section_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class CourseSectionEvent < LiveEvents::BaseEvent
+  class CourseSectionEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       section = Section.where(canvas_id: local_canvas_id(payload[:course_section_id])).first_or_initialize

data/lib/canvas_sync/generators/templates/services/live_events/enrollment_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class EnrollmentEvent < LiveEvents::BaseEvent
+  class EnrollmentEvent < CanvasSync::LiveEvents::BaseHandler
     attr_accessor :enrollment
 
     def process()

data/lib/canvas_sync/generators/templates/services/live_events/grade_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class GradeEvent < LiveEvents::BaseEvent
+  class GradeEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       raise "process must be implemented in your subclass"

data/lib/canvas_sync/generators/templates/services/live_events/module_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class ModuleEvent < LiveEvents::BaseEvent
+  class ModuleEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       return unless payload["context_type"] == "Course"

data/lib/canvas_sync/generators/templates/services/live_events/module_item_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class ModuleItemEvent < LiveEvents::BaseEvent
+  class ModuleItemEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       context_module_item = ContextModuleItem.find_or_initialize_by(canvas_id: payload["module_item_id"])

data/lib/canvas_sync/generators/templates/services/live_events/submission_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class SubmissionEvent < LiveEvents::BaseEvent
+  class SubmissionEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       submission = Submission.where(canvas_id: local_canvas_id(payload["submission_id"])).first_or_initialize

data/lib/canvas_sync/generators/templates/services/live_events/syllabus_event.rb

@@ -1,7 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class SyllabusEvent < LiveEvents::BaseEvent
+  class SyllabusEvent < CanvasSync::LiveEvents::BaseHandler
 
     def process
       # syllabus = Syllabus.where(course_id: attrs[:course_id]).first_or_initialize

data/lib/canvas_sync/generators/templates/services/live_events/user_event.rb

@@ -1,8 +1,7 @@
 # <%= autogenerated_event_warning %>
 
 module LiveEvents
-  class UserEvent < LiveEvents::BaseEvent
-
+  class UserEvent < CanvasSync::LiveEvents::BaseHandler
     def process
       canvas_user_id = local_canvas_id(payload[:user_id])
       user = User.where(canvas_id: canvas_user_id).first_or_initialize
@@ -11,7 +10,6 @@ module LiveEvents
       # This can happen when a new user is created, but hasn't setup a login on Canvas yet.
       Rails.logger.info("Failed to fetch user #{canvas_user_id}: #{e.backtrace}")
     end
-
   end
 
   class UserCreatedEvent < LiveEvents::UserEvent; end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -23,40 +23,70 @@ module CanvasSync
         end
       end
 
-      def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
-        csv_column_names = mapping.keys
-        database_column_names = mapping.values.map { |value| value[:database_column_name] }
-        rows = []
-        row_ids = {}
-        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
+      def self.perform_in_batches(report_file_path, raw_mapping, klass, conflict_target, import_args: {})
+        mapping = {}.with_indifferent_access
+        raw_mapping.each do |db_col, opts|
+          next if opts[:deprecated] && !klass.column_names.include?(db_col.to_s)
 
-        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
-          row = yield(row) if block_given?
-          next if row.nil?
+          mapping[db_col] = opts
+        end
 
-          if conflict_target
-            next if row_ids[row[conflict_target]]
-            row_ids[row[conflict_target]] = true
-          end
+        csv_column_names = mapping.values.map { |value| value[:report_column].to_s }
+        database_column_names = mapping.keys
+
+        conflict_target = Array(conflict_target).map(&:to_s)
+        conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
-          rows << csv_column_names.map do |column|
-            if mapping[column][:type].to_sym == :datetime
-              # TODO: add some timezone config to the mapping.
-              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
-              DateTime.parse(row[column]).utc rescue nil # rubocop:disable Style/RescueModifier
-            else
-              row[column]
+        row_ids = {}
+        batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
+          row_ids = {}
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
+        end
+
+        row_buffer_out = ->(row) {
+          formatted_row = mapping.map do |db_col, col_def|
+            value = nil
+            value = row[col_def[:report_column]] if col_def[:report_column]
+
+            if col_def[:type]
+              if col_def[:type].to_sym == :datetime
+                # TODO: add some timezone config to the mapping.
+                # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+                value = DateTime.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+              end
             end
+
+            value = col_def[:transform].call(value, row) if col_def[:transform]
+
+            value
           end
 
-          if rows.length >= batch_size
-            perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
-            rows = []
-            row_ids = {}
+          if conflict_target.present?
+            key = conflict_target_indices.map{|ct| formatted_row[ct] }
+            next if row_ids[key]
+
+            row_ids[key] = true
           end
+
+          batcher << formatted_row
+        }
+
+        row_buffer = nil
+        if defined?(User) && klass == User && csv_column_names.include?('user_id')
+          row_buffer = UserRowBuffer.new(&row_buffer_out)
+        else
+          row_buffer = NullRowBuffer.new(&row_buffer_out)
         end
 
-        perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
+        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
+          row = yield(row) if block_given?
+          next if row.nil?
+
+          row_buffer << row
+        end
+
+        row_buffer.flush
+        batcher.flush
       end
 
       def self.perform_import(klass, columns, rows, conflict_target, import_args={})
@@ -64,15 +94,35 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
-        klass.import(columns, rows, options)
+
+        result = nil
+        callback_env = {
+          batch: rows,
+          import_result: nil,
+          import_options: options,
+        }
+        ClassCallbackExecutor.run_if_defined(klass, :sync_batch_import, callback_env) do
+          result = klass.import(columns, rows, options)
+          callback_env[:import_result] = result
+
+          global_updates = {
+            canvas_synced_at: DateTime.now,
+            canvas_sync_batch_id: JobBatches::Batch.current_context[:sync_batch_id],
+          }
+          global_updates.slice!(*klass.column_names.map(&:to_sym))
+          if global_updates.present? && result.ids.present?
+            klass.where(id: result.ids).update_all(global_updates)
+          end
+        end
+
+        result
       end
 
       # This method generates SQL that looks like:
@@ -85,16 +135,73 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start = nil)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+
+        if klass.column_names.include?("canvas_synced_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.canvas_synced_at < '#{report_start}'"
+        elsif klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+
+        condition_sql
       end
 
       def self.batch_size
         batch_size = ENV["BULK_IMPORTER_BATCH_SIZE"].to_i
         batch_size > 0 ? batch_size : DEFAULT_BATCH_SIZE
       end
+
+      class RowBuffer
+        def initialize(&block)
+          @flush_out = block
+          @buffered_rows = []
+        end
+
+        def <<(v)
+          @buffered_rows << v
+        end
+
+        def flush(value = @buffered_rows)
+          if value.is_a?(Array)
+            value.each do |v|
+              @flush_out.call(v)
+            end
+          else
+            @flush_out.call(value)
+          end
+          @buffered_rows = []
+        end
+      end
+
+      class NullRowBuffer
+        def initialize(&block)
+          @flush_out = block
+        end
+
+        def <<(v)
+          @flush_out.call(v)
+        end
+
+        def flush; end
+      end
+
+      # Ensures that, if a User has multiple rows, one with a SIS ID is preferred.
+      # This is mainly to fix issues in legacy apps - the suggested approach for new apps
+      # is to sync and use the Pseudonymes table
+      class UserRowBuffer < RowBuffer
+        def <<(v)
+          flush if @buffered_rows[0] && @buffered_rows[0][:canvas_user_id] != v[:canvas_user_id]
+          super
+        end
+
+        def flush
+          row = @buffered_rows.find{|r| r[:user_id].present? } || @buffered_rows.last
+          super(row.present? ? [row] : [])
+        end
+      end
     end
   end
 end

data/lib/canvas_sync/job.rb

@@ -3,7 +3,11 @@ require "active_job"
 module CanvasSync
   # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
   class Job < ActiveJob::Base
+    attr_reader :job_log
+
     before_enqueue do |job|
+      # Fixes an issue when the Job is enqueued by SidekiqScheduler/Rufus.
+      ActiveRecord::Base.connection.verify!
       create_job_log(job)
     end
 
@@ -13,8 +17,6 @@ module CanvasSync
       @job_log.started_at = Time.now
       @job_log.save
 
-      @job_chain = job.arguments[0] if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
-
       begin
         block.call
         @job_log.status = JobLog::SUCCESS_STATUS
@@ -22,11 +24,11 @@ module CanvasSync
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace.join('\n')
         @job_log.status = JobLog::ERROR_STATUS
-        if @job_chain&.[](:global_options)&.[](:on_failure)&.present?
+        if batch_context&.[](:on_failure)&.present?
           begin
-            class_name, method = @job_chain[:global_options][:on_failure].split('.')
+            class_name, method = batch_context[:on_failure].split('.')
             klass = class_name.constantize
-            klass.send(method.to_sym, e, job_chain: @job_chain, job_log: @job_log)
+            klass.send(method.to_sym, e, batch_context: batch_context, job_log: @job_log)
           rescue => e2
             @job_log.backtrace += "\n\nError Occurred while handling an Error: #{e2.class}: #{e2.message}"
             @job_log.backtrace += "\n" + e2.backtrace.join('\n')

data/lib/canvas_sync/job_batches/active_job.rb

@@ -0,0 +1,108 @@
+
+module CanvasSync
+  module JobBatches
+    module ActiveJob
+      module BatchAwareJob
+        extend ActiveSupport::Concern
+
+        included do
+          around_perform do |job, block|
+            if (@bid) # This _must_ be @bid - not just bid
+              prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
+              begin
+                Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
+                block.call
+                Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
+                Batch.process_successful_job(@bid, job_id)
+              rescue
+                Batch.process_failed_job(@bid, job_id)
+                raise
+              ensure
+                Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
+              end
+            else
+              block.call
+            end
+          end
+
+          around_enqueue do |job, block|
+            if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
+              @bid = batch.bid
+              batch.increment_job_queue(job_id) if @bid
+            end
+            block.call
+          end
+        end
+
+        def bid
+          @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
+        end
+
+        def batch
+          Thread.current[CURRENT_BATCH_THREAD_KEY]
+        end
+
+        def batch_context
+          batch&.context || {}
+        end
+
+        def valid_within_batch?
+          batch.valid?
+        end
+
+        def serialize
+          super.tap do |data|
+            data['batch_id'] = @bid # This _must_ be @bid - not just bid
+            data
+          end
+        end
+
+        def deserialize(data)
+          super
+          @bid = data['batch_id']
+        end
+      end
+
+      class ActiveJobCallbackWorker < ::ActiveJob::Base
+        include Batch::Callback::CallbackWorkerCommon
+
+        def self.enqueue_all(args, queue)
+          args.each do |arg_set|
+            set(queue: queue).perform_later(*arg_set)
+          end
+        end
+      end
+
+      def self.handle_job_death(job, error = nil)
+        if job.is_a?(Array)
+          event = ActiveSupport::Notifications::Event.new(*job)
+          payload = event.payload
+          job = payload[:job].serialize
+          error = payload[:error]
+        end
+
+        if job["job_id"].present? && job["batch_id"].present?
+          CanvasSync::JobBatches::Batch.process_dead_job(job['batch_id'], job['job_id'])
+        end
+      end
+
+      def self.configure
+        ::ActiveJob::Base.include BatchAwareJob
+
+        begin
+          ActiveSupport::Notifications.subscribe "discard.active_job" do |*args|
+            handle_job_death(args)
+          end
+
+          ActiveSupport::Notifications.subscribe "retry_stopped.active_job" do |*args|
+            handle_job_death(args)
+          end
+        rescue => err
+          Rails.logger.warn(err)
+        end
+
+        Batch::Callback.worker_class ||= ActiveJobCallbackWorker
+      end
+    end
+  end
+end