canvas_sync 0.15.1 → 0.16.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 498fd2c3e3801c477077eced06fda11055ef8924
-  data.tar.gz: 2a8b2979721246d70bd29fbb7a7a596480fb3b8e
+SHA256:
+  metadata.gz: 9be3b81c22d5b45a02d88fa29b5d57afa152ee7dec6ce0ecf3d81c041507c619
+  data.tar.gz: 0c42577064cbf018f8fd8d5f665e1adfbc542e38e704a675e735491543b3ab40
 SHA512:
-  metadata.gz: e645ffde592d741f67abddf9067816f4e2c440c7fc44d7af344def28dcc0c277f0573855bd0dceb9e4b883be830516b210d5ef43be79679c5a4bab42f3acb41e
-  data.tar.gz: c415bffe6b53ca6fc92202119300ff71cf39d8cbfeec7d0387104bcfe1777740c1085d0e2e5ef2694ab7fc9dc3a99346ceda1470c148d844178804622c640d1d
+  metadata.gz: 9ea295b8cc43aa6b27bf6067cf684d024430a9fd85608bdc0a243113920b1b65b72c257e6a99644afb5d20455ecf2265949ce941a9ae1d9bea160ab1bee803ba
+  data.tar.gz: e63b7f557e792cb850a5799753f9effe436ad391bc6e4442768683b8c4d7c8eea1d1e9bbd6419127f2f4401f6cce6129019fb73192496f7efc1d731d7ef48847

data/db/migrate/20170915210836_create_canvas_sync_job_log.rb

@@ -1,35 +1,16 @@
-if Rails.version.to_f >= 5.0
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
+class CreateCanvasSyncJobLog < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_job_logs do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :exception
+      t.text :backtrace
+      t.string :job_class
+      t.string :status
+      t.text :metadata
+      t.text :job_arguments
 
-        t.timestamps
-      end
-    end
-  end
-else
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
-
-        t.timestamps
-      end
+      t.timestamps
     end
   end
 end

data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb

@@ -1,15 +1,6 @@
-if Rails.version.to_f >= 5.0
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
-  end
-else
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
+class AddJobIdToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :job_id, :string
+    add_index :canvas_sync_job_logs, :job_id
   end
 end

data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb

@@ -1,13 +1,5 @@
-if Rails.version.to_f >= 5.0
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
-  end
-else
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
+class AddForkCountToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :fork_count, :integer
   end
 end

data/lib/canvas_sync.rb

@@ -2,6 +2,7 @@ require "bearcat"
 
 require "canvas_sync/version"
 require "canvas_sync/engine"
+require "canvas_sync/misc_helper"
 require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
 require "canvas_sync/job_chain"
@@ -99,6 +100,7 @@ module CanvasSync
       invoke_next(job_chain)
     end
 
+    # @deprecated
     def duplicate_chain(job_chain)
       Marshal.load(Marshal.dump(job_chain))
     end
@@ -109,35 +111,13 @@ module CanvasSync
     #
     # @param job_chain [Hash] A chain of jobs to execute
     def invoke_next(job_chain, extra_options: {})
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      return if job_chain[:jobs].empty?
-
-      # Make sure all job classes are serialized as strings
-      job_chain[:jobs].each { |job| job[:job] = job[:job].to_s }
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      jobs = duped_job_chain[:jobs]
-      next_job = jobs.shift
-      next_job_class = next_job[:job].constantize
-      next_options = next_job[:options] || {}
-      next_options.merge!(extra_options)
-      next_job_class.perform_later(duped_job_chain, next_options)
+      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
+      job_chain.perform_next(extra_options)
     end
 
-    def fork(job_log, job_chain, keys: [])
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      duped_job_chain[:global_options][:fork_path] ||= []
-      duped_job_chain[:global_options][:fork_keys] ||= []
-      duped_job_chain[:global_options][:fork_path] << job_log.job_id
-      duped_job_chain[:global_options][:fork_keys] << ['canvas_term_id']
-      duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
-      sub_items = yield duped_job_chain
-      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
-      job_log.fork_count = sub_count
-      sub_items
+    def fork(job_log, job_chain, keys: [], &blk)
+      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
+      job_chain.fork(job_log, keys: keys, &blk)
     end
 
     # Given a Model or Relation, scope it down to items that should be synced
@@ -146,6 +126,10 @@ module CanvasSync
       terms.each do |t|
         return scope.send(t) if scope.respond_to?(t)
       end
+      model = scope.try(:model) || scope
+      if model.try(:column_names)&.include?(:workflow_state)
+        return scope.where.not(workflow_state: %w[deleted])
+      end
       Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
       scope
     end

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -29,6 +29,33 @@ module CanvasSync::Concerns
         end
       end
 
+      def bulk_sync_from_api_result(api_array, conflict_target: :canvas_id, import_args: {}, all_pages: true, batch_size: 1000)
+        columns = api_sync_options.keys
+
+        update_conditions = {
+          condition: Importers::BulkImporter.condition_sql(self, columns),
+          columns: columns,
+        }
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
+        options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
+
+        if all_pages
+          batcher = BatchProcessor.new(of: batch_size) do |batch|
+            import(columns, batch, options)
+          end
+          api_array.all_pages_each do |api_item|
+            item = new.assign_from_api_params(api_items)
+            batcher << item
+          end
+          batcher.flush
+        else
+          items = api_array.map do |api_item|
+            new.assign_from_api_params(api_items)
+          end
+          import(columns, batch, options)
+        end
+      end
+
       def api_sync_options=(opts)
         @api_sync_options = opts
       end

data/lib/canvas_sync/concerns/legacy_columns.rb

@@ -15,13 +15,14 @@ module CanvasSync::Concerns
       private
 
       def legacy_column_apply(cls)
-        cid_column = "canvas_#{subclass.name.downcase}_id"
-        column_names = subclass.columns.map(&:name)
+        return if cls.abstract_class
+        cid_column = "canvas_#{cls.name.downcase}_id"
+        column_names = cls.columns.map(&:name)
         return if column_names.include?('canvas_id') && column_names.include?(cid_column)
         if column_names.include?('canvas_id')
-          subclass.alias_attribute(cid_column.to_sym, :canvas_id)
+          cls.alias_attribute(cid_column.to_sym, :canvas_id)
         elsif column_names.include?(cid_column)
-          subclass.alias_attribute(:canvas_id, cid_column.to_sym)
+          cls.alias_attribute(:canvas_id, cid_column.to_sym)
         end
       rescue ActiveRecord::StatementInvalid
       end

data/lib/canvas_sync/generators/templates/migrations/create_submissions.rb

@@ -8,6 +8,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
       t.bigint :canvas_assignment_id
       t.bigint :canvas_user_id
       t.datetime :submitted_at
+      t.datetime :due_at
       t.datetime :graded_at
       t.float :score
       t.float :points_possible

data/lib/canvas_sync/generators/templates/models/account.rb

@@ -14,6 +14,9 @@ class Account < ApplicationRecord
                           primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
   has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
 
+  scope :active, -> { where.not(workflow_state: 'deleted') }
+  # scope :should_canvas_sync, -> { active } # Optional - uses .active if not given
+
   api_syncable({
     name: :name,
     workflow_state: :workflow_state,

data/lib/canvas_sync/generators/templates/models/submission.rb

@@ -14,6 +14,7 @@ class Submission < ApplicationRecord
     canvas_user_id: :user_id,
     submitted_at: :submitted_at,
     graded_at: :graded_at,
+    cached_due_date: :due_at,
     score: :score,
     excused: :excused,
     workflow_state: :workflow_state,

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job.rb

@@ -3,6 +3,8 @@ require "active_job"
 module CanvasSync
   # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
   class Job < ActiveJob::Base
+    attr_reader :job_chain, :job_log
+
     before_enqueue do |job|
       create_job_log(job)
     end
@@ -13,7 +15,11 @@ module CanvasSync
       @job_log.started_at = Time.now
       @job_log.save
 
-      @job_chain = job.arguments[0] if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
+      if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
+        # @job_chain = JobChain.new(job.arguments[0])
+        @job_chain = job.arguments[0]
+        job.arguments[0] = @job_chain
+      end
 
       begin
         block.call
@@ -22,7 +28,7 @@ module CanvasSync
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace.join('\n')
         @job_log.status = JobLog::ERROR_STATUS
-        if @job_chain&.[](:global_options)&.[](:on_failure)&.present?
+        if @job_chain&.dig(:global_options, :on_failure)&.present?
           begin
             class_name, method = @job_chain[:global_options][:on_failure].split('.')
             klass = class_name.constantize

data/lib/canvas_sync/job_chain.rb

@@ -45,7 +45,52 @@ module CanvasSync
     end
 
     def process!(extra_options: {})
-      CanvasSync::invoke_next(self, extra_options: extra_options)
+      perform_next(extra_options)
+    end
+
+    def duplicate
+      self.class.new(Marshal.load(Marshal.dump(chain_data)))
+    end
+
+    def normalize!
+      @chain_data[:global_options] ||= {}
+    end
+
+    def serialize
+      normalize!
+      chain_data
+    end
+
+    def perform_next(extra_options = {})
+      return if jobs.empty?
+
+      # Make sure all job classes are serialized as strings
+      jobs.each { |job| job[:job] = job[:job].to_s }
+
+      duped_job_chain = duplicate
+
+      jobs = duped_job_chain[:jobs]
+      next_job = jobs.shift
+      next_job_class = next_job[:job].constantize
+      next_options = next_job[:options] || {}
+      next_options.merge!(extra_options)
+      next_job_class.perform_later(duped_job_chain.serialize, next_options)
+    end
+
+    def fork(job_log, keys: [])
+      duped_job_chain = duplicate
+      duped_job_chain[:fork_state] ||= {}
+      duped_job_chain[:fork_state][:forking_path] ||= []
+      duped_job_chain[:fork_state][:pre_fork_globals] ||= []
+
+      duped_job_chain[:fork_state][:forking_path] << job_log.job_id
+      duped_job_chain[:fork_state][:pre_fork_globals] << global_options
+      # duped_job_chain[:global_options][:on_failure] ||= ['CanvasSync::Jobs::ForkGather.handle_branch_error']
+
+      sub_items = yield duped_job_chain
+      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
+      job_log.update!(fork_count: sub_count)
+      sub_items
     end
 
     private

data/lib/canvas_sync/jobs/fork_gather.rb

@@ -4,25 +4,40 @@ module CanvasSync
       def perform(job_chain, options)
         forked_job = self.class.forked_at_job(job_chain)
 
-        if forked_job.present?
-          forked_job.with_lock do
-            forked_job.fork_count -= 1
-            forked_job.save!
-          end
+        while true
+          if forked_job.present?
+            forked_job.with_lock do
+              forked_job.fork_count -= 1
+              forked_job.save!
+            end
+
+            if forked_job.fork_count <= 0
+              pfgs = job_chain[:fork_state][:pre_fork_globals].pop
+              job_chain[:global_options] = pfgs
 
-          if forked_job.fork_count <= 0
-            (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
-              job_chain[:global_options].delete(k.to_sym)
+              if options[:gather_all]
+                # If we want to gather all, repeat for the next level fork
+                forked_job = self.class.forked_at_job(job_chain)
+              else
+                forked_job = nil
+              end
+            else
+              # If a fork was found and it isn't complete, break the loop before continuing the chain
+              break
             end
-            CanvasSync.invoke_next(job_chain)
+
+            # Repeat this logic for [if gather_all] the next fork up, or [if not gather_all] nil
+            next
           end
-        else
+
+          # If there is no current fork (either not in a fork, or all forks were closed), continue the chain
           CanvasSync.invoke_next(job_chain)
+          break
         end
       end
 
       def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
-        return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
+        return nil unless job_chain&.dig(:fork_state, :forking_path).present?
 
         duped_chain = CanvasSync.duplicate_chain(job_chain)
         job_list = duped_chain[:jobs]
@@ -46,7 +61,7 @@ module CanvasSync
       protected
 
       def self.forked_at_job(job_chain)
-        fork_item = (job_chain[:global_options][:fork_path] || []).pop
+        fork_item = (job_chain.dig(:fork_state, :forking_path) || []).pop
 
         if fork_item.present?
           CanvasSync::JobLog.find_by(job_id: fork_item)

data/lib/canvas_sync/jobs/report_starter.rb

@@ -12,7 +12,7 @@ module CanvasSync
       # @return [nil]
       def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-
+	options[:sync_start_time] = DateTime.now.utc.iso8601
         report_id = if allow_redownloads
                       get_cached_report(job_chain, account_id, report_name, report_params)
                     else

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -8,15 +8,15 @@ module CanvasSync
       #   models to sync.
       def perform(job_chain, options)
         if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
+          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
             Term.send(options[:term_scope]).find_each.map do |term|
+              fork = fork_template.duplicate
               # Deep copy the job_chain so each report gets the correct term id passed into
               # its options with no side effects
               term_id = get_term_id(term)
-              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-              duped_job_chain[:global_options][:canvas_term_id] = term_id
+              fork[:global_options][:canvas_term_id] = term_id
               {
-                job_chain: duped_job_chain,
+                job_chain: fork.serialize,
                 params: report_params(options, term_id),
                 options: options,
               }
@@ -31,7 +31,7 @@ module CanvasSync
         end
       end
 
-      private
+      protected
 
       def start_report(report_params, job_chain, options)
         CanvasSync::Jobs::ReportStarter.perform_later(