canvas_sync 0.16.4 → 0.16.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ddf6a257acf95c92ba99a6d03c9bcdd99ba1a8ff
-  data.tar.gz: f10e8f91a225462545354a24daaf006a170bd90d
+SHA256:
+  metadata.gz: 9be3b81c22d5b45a02d88fa29b5d57afa152ee7dec6ce0ecf3d81c041507c619
+  data.tar.gz: 0c42577064cbf018f8fd8d5f665e1adfbc542e38e704a675e735491543b3ab40
 SHA512:
-  metadata.gz: cde3a3a534316a0faa535347a6c71aa63ab46c5cc5d9c21b01512bb623bed7f6b34fc7da38de345fefcf238cc44f0423f849dd08aa8f57f292472838aa6f7d0c
-  data.tar.gz: 2e1b8217af038166f60afe4032f0f1ec806761fb81d5727aec57540919146335f7a28292e8e81ba87220eb2d687325b8145be798ed6a29b07aaad0cd6e3b56f4
+  metadata.gz: 9ea295b8cc43aa6b27bf6067cf684d024430a9fd85608bdc0a243113920b1b65b72c257e6a99644afb5d20455ecf2265949ce941a9ae1d9bea160ab1bee803ba
+  data.tar.gz: e63b7f557e792cb850a5799753f9effe436ad391bc6e4442768683b8c4d7c8eea1d1e9bbd6419127f2f4401f6cce6129019fb73192496f7efc1d731d7ef48847

data/db/migrate/20170915210836_create_canvas_sync_job_log.rb

@@ -1,35 +1,16 @@
-if Rails.version.to_f >= 5.0
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
+class CreateCanvasSyncJobLog < CanvasSync::MiscHelper::MigrationClass
+  def change
+    create_table :canvas_sync_job_logs do |t|
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.string :exception
+      t.text :backtrace
+      t.string :job_class
+      t.string :status
+      t.text :metadata
+      t.text :job_arguments
 
-        t.timestamps
-      end
-    end
-  end
-else
-  class CreateCanvasSyncJobLog < ActiveRecord::Migration
-    def change
-      create_table :canvas_sync_job_logs do |t|
-        t.datetime :started_at
-        t.datetime :completed_at
-        t.string :exception
-        t.text :backtrace
-        t.string :job_class
-        t.string :status
-        t.text :metadata
-        t.text :job_arguments
-
-        t.timestamps
-      end
+      t.timestamps
     end
   end
 end

data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb

@@ -1,15 +1,6 @@
-if Rails.version.to_f >= 5.0
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
-  end
-else
-  class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :job_id, :string
-      add_index :canvas_sync_job_logs, :job_id
-    end
+class AddJobIdToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :job_id, :string
+    add_index :canvas_sync_job_logs, :job_id
   end
 end

data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb

@@ -1,13 +1,5 @@
-if Rails.version.to_f >= 5.0
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
-  end
-else
-  class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration
-    def change
-      add_column :canvas_sync_job_logs, :fork_count, :integer
-    end
+class AddForkCountToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_job_logs, :fork_count, :integer
   end
 end

data/lib/canvas_sync.rb

@@ -2,6 +2,7 @@ require "bearcat"
 
 require "canvas_sync/version"
 require "canvas_sync/engine"
+require "canvas_sync/misc_helper"
 require "canvas_sync/class_callback_executor"
 require "canvas_sync/job"
 require "canvas_sync/job_chain"
@@ -99,6 +100,7 @@ module CanvasSync
       invoke_next(job_chain)
     end
 
+    # @deprecated
     def duplicate_chain(job_chain)
       Marshal.load(Marshal.dump(job_chain))
     end
@@ -109,35 +111,13 @@ module CanvasSync
     #
     # @param job_chain [Hash] A chain of jobs to execute
     def invoke_next(job_chain, extra_options: {})
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      return if job_chain[:jobs].empty?
-
-      # Make sure all job classes are serialized as strings
-      job_chain[:jobs].each { |job| job[:job] = job[:job].to_s }
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      jobs = duped_job_chain[:jobs]
-      next_job = jobs.shift
-      next_job_class = next_job[:job].constantize
-      next_options = next_job[:options] || {}
-      next_options.merge!(extra_options)
-      next_job_class.perform_later(duped_job_chain, next_options)
+      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
+      job_chain.perform_next(extra_options)
     end
 
-    def fork(job_log, job_chain, keys: [])
-      job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
-
-      duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-      duped_job_chain[:global_options][:fork_path] ||= []
-      duped_job_chain[:global_options][:fork_keys] ||= []
-      duped_job_chain[:global_options][:fork_path] << job_log.job_id
-      duped_job_chain[:global_options][:fork_keys] << keys.map(&:to_s)
-      duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
-      sub_items = yield duped_job_chain
-      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
-      job_log.fork_count = sub_count
-      sub_items
+    def fork(job_log, job_chain, keys: [], &blk)
+      job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
+      job_chain.fork(job_log, keys: keys, &blk)
     end
 
     # Given a Model or Relation, scope it down to items that should be synced

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job.rb

@@ -3,6 +3,8 @@ require "active_job"
 module CanvasSync
   # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
   class Job < ActiveJob::Base
+    attr_reader :job_chain, :job_log
+
     before_enqueue do |job|
       create_job_log(job)
     end
@@ -13,7 +15,11 @@ module CanvasSync
       @job_log.started_at = Time.now
       @job_log.save
 
-      @job_chain = job.arguments[0] if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
+      if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
+        # @job_chain = JobChain.new(job.arguments[0])
+        @job_chain = job.arguments[0]
+        job.arguments[0] = @job_chain
+      end
 
       begin
         block.call
@@ -22,7 +28,7 @@ module CanvasSync
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace.join('\n')
         @job_log.status = JobLog::ERROR_STATUS
-        if @job_chain&.[](:global_options)&.[](:on_failure)&.present?
+        if @job_chain&.dig(:global_options, :on_failure)&.present?
           begin
             class_name, method = @job_chain[:global_options][:on_failure].split('.')
             klass = class_name.constantize

data/lib/canvas_sync/job_chain.rb

@@ -45,7 +45,52 @@ module CanvasSync
     end
 
     def process!(extra_options: {})
-      CanvasSync::invoke_next(self, extra_options: extra_options)
+      perform_next(extra_options)
+    end
+
+    def duplicate
+      self.class.new(Marshal.load(Marshal.dump(chain_data)))
+    end
+
+    def normalize!
+      @chain_data[:global_options] ||= {}
+    end
+
+    def serialize
+      normalize!
+      chain_data
+    end
+
+    def perform_next(extra_options = {})
+      return if jobs.empty?
+
+      # Make sure all job classes are serialized as strings
+      jobs.each { |job| job[:job] = job[:job].to_s }
+
+      duped_job_chain = duplicate
+
+      jobs = duped_job_chain[:jobs]
+      next_job = jobs.shift
+      next_job_class = next_job[:job].constantize
+      next_options = next_job[:options] || {}
+      next_options.merge!(extra_options)
+      next_job_class.perform_later(duped_job_chain.serialize, next_options)
+    end
+
+    def fork(job_log, keys: [])
+      duped_job_chain = duplicate
+      duped_job_chain[:fork_state] ||= {}
+      duped_job_chain[:fork_state][:forking_path] ||= []
+      duped_job_chain[:fork_state][:pre_fork_globals] ||= []
+
+      duped_job_chain[:fork_state][:forking_path] << job_log.job_id
+      duped_job_chain[:fork_state][:pre_fork_globals] << global_options
+      # duped_job_chain[:global_options][:on_failure] ||= ['CanvasSync::Jobs::ForkGather.handle_branch_error']
+
+      sub_items = yield duped_job_chain
+      sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
+      job_log.update!(fork_count: sub_count)
+      sub_items
     end
 
     private

data/lib/canvas_sync/jobs/fork_gather.rb

@@ -4,25 +4,40 @@ module CanvasSync
       def perform(job_chain, options)
         forked_job = self.class.forked_at_job(job_chain)
 
-        if forked_job.present?
-          forked_job.with_lock do
-            forked_job.fork_count -= 1
-            forked_job.save!
-          end
+        while true
+          if forked_job.present?
+            forked_job.with_lock do
+              forked_job.fork_count -= 1
+              forked_job.save!
+            end
+
+            if forked_job.fork_count <= 0
+              pfgs = job_chain[:fork_state][:pre_fork_globals].pop
+              job_chain[:global_options] = pfgs
 
-          if forked_job.fork_count <= 0
-            (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
-              job_chain[:global_options].delete(k.to_sym)
+              if options[:gather_all]
+                # If we want to gather all, repeat for the next level fork
+                forked_job = self.class.forked_at_job(job_chain)
+              else
+                forked_job = nil
+              end
+            else
+              # If a fork was found and it isn't complete, break the loop before continuing the chain
+              break
             end
-            CanvasSync.invoke_next(job_chain)
+
+            # Repeat this logic for [if gather_all] the next fork up, or [if not gather_all] nil
+            next
           end
-        else
+
+          # If there is no current fork (either not in a fork, or all forks were closed), continue the chain
           CanvasSync.invoke_next(job_chain)
+          break
         end
       end
 
       def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
-        return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
+        return nil unless job_chain&.dig(:fork_state, :forking_path).present?
 
         duped_chain = CanvasSync.duplicate_chain(job_chain)
         job_list = duped_chain[:jobs]
@@ -46,7 +61,7 @@ module CanvasSync
       protected
 
       def self.forked_at_job(job_chain)
-        fork_item = (job_chain[:global_options][:fork_path] || []).pop
+        fork_item = (job_chain.dig(:fork_state, :forking_path) || []).pop
 
         if fork_item.present?
           CanvasSync::JobLog.find_by(job_id: fork_item)

data/lib/canvas_sync/jobs/report_starter.rb

@@ -12,7 +12,7 @@ module CanvasSync
       # @return [nil]
       def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-
+	options[:sync_start_time] = DateTime.now.utc.iso8601
         report_id = if allow_redownloads
                       get_cached_report(job_chain, account_id, report_name, report_params)
                     else

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -8,15 +8,15 @@ module CanvasSync
       #   models to sync.
       def perform(job_chain, options)
         if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
+          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
             Term.send(options[:term_scope]).find_each.map do |term|
+              fork = fork_template.duplicate
               # Deep copy the job_chain so each report gets the correct term id passed into
               # its options with no side effects
               term_id = get_term_id(term)
-              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-              duped_job_chain[:global_options][:canvas_term_id] = term_id
+              fork[:global_options][:canvas_term_id] = term_id
               {
-                job_chain: duped_job_chain,
+                job_chain: fork.serialize,
                 params: report_params(options, term_id),
                 options: options,
               }

data/lib/canvas_sync/jobs/sync_simple_table_job.rb

@@ -9,15 +9,15 @@ module CanvasSync
       # @param options [Hash]
       def perform(job_chain, options)
         if options[:term_scope]
-          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
+          sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
             Term.send(options[:term_scope]).find_each.map do |term|
+              fork = fork_template.duplicate
               # Deep copy the job_chain so each report gets the correct term id passed into
               # its options with no side effects
               term_id = get_term_id(term)
-              duped_job_chain = Marshal.load(Marshal.dump(job_chain))
-              duped_job_chain[:global_options][:canvas_term_id] = term_id
+              fork[:global_options][:canvas_term_id] = term_id
               {
-                job_chain: duped_job_chain,
+                job_chain: fork.serialize,
                 params: report_params(options, term_id),
                 options: options,
               }

data/lib/canvas_sync/misc_helper.rb

@@ -0,0 +1,15 @@
+require 'active_record'
+
+module CanvasSync
+  module MiscHelper
+    MigrationClass = Rails.version < '5.0' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+
+    def self.to_boolean(v)
+      if Rails.version < '5.0'
+        ActiveRecord::Type::Boolean.new.type_cast_from_user(v)
+      else
+        ActiveRecord::Type::Boolean.new.deserialize(v)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end