canvas_sync 0.17.0.beta13 → 0.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7b76bf6c25022960d7ca9605aac613424a763171df506e19bb71ddfe7afac9c
-  data.tar.gz: 37919cedb89d90f6589cb049775c9a7399c7d94a5c95aeb8a5b40f02de27496d
+  metadata.gz: e7e920904fe92285bd67cbf48a88e8aa6c03761dbb543eb9ee477058f3086c2a
+  data.tar.gz: d2221d37737fde81de62a0f9d6d304cf059c30cc78f3be814c01849365257680
 SHA512:
-  metadata.gz: ef40e571c7b24b98102a6278f656fcb98f9f3ec960c3951df3152601cb7dedbee554991bfa5f073fb32a5ac9424da89bdab1030afe0b73b68f41c203babf6771
-  data.tar.gz: 674b086a5b0b5ab0e2aaefa0aa3df4a7c3efd437db171f20c96b09005b3c914e7d9abec5df215b838f07515fa7d9abba54f046b347f0a4e1b4137f0e14f1e348
+  metadata.gz: 3a988e7678bc3f0e6e33325171f7665cc016cf30d07aa299c69268339587433e69d7c11036d797358c4f563d21956b74a48bb7bd5cc656d86ba01edde5d85a3f
+  data.tar.gz: 74f9f9ba4a6a6a59e1a7dbb2dba351cee464d12d74ef526784fa63d81e319656163ccb52fbca1f71a7af664148f6712e77513bbf4346e432bc9394fede53ed3f

data/lib/canvas_sync.rb

@@ -42,6 +42,22 @@ module CanvasSync
     xlist
   ].freeze
 
+  SUPPORTED_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
+  DEFAULT_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
   SUPPORTED_LIVE_EVENTS = %w[
     course
     enrollment
@@ -109,6 +125,7 @@ module CanvasSync
     def default_provisioning_report_chain(
       models,
       term_scope: nil,
+      term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
       legacy_support: false,
       account_id: nil,
       updated_after: nil,
@@ -165,6 +182,10 @@ module CanvasSync
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
 
+      (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
+        try_add_model_job.call(mdl)
+      end
+
       ###############################
       # Per-term provisioning jobs
       ###############################
@@ -174,11 +195,9 @@ module CanvasSync
       current_chain << per_term_chain
       current_chain = per_term_chain
 
-      try_add_model_job.call('assignments')
-      try_add_model_job.call('submissions')
-      try_add_model_job.call('assignment_groups')
-      try_add_model_job.call('context_modules')
-      try_add_model_job.call('context_module_items')
+      term_scoped_models.each do |mdl|
+        try_add_model_job.call(mdl)
+      end
 
       current_chain.insert(
         generate_provisioning_jobs(models, options, only_split: ['users'])

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -60,10 +60,12 @@ module CanvasSync
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          parent_job, sub_index = matching_jobs[0]
-          chain = self.class.get_chain_parameter(parent_job)
+          relative_job, sub_index = matching_jobs[0]
+          parent_job = find_parent_job(relative_job)
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
+          chain = self.class.get_chain_parameter(parent_job)
+
           if parent_job[:job] != needed_parent_type
             old_job = chain[sub_index]
             parent_job = chain[sub_index] = {
@@ -129,6 +131,24 @@ module CanvasSync
         end
       end
 
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
       class << self
         def _job_type_definitions
           @job_type_definitions ||= {}

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -6,7 +6,9 @@ module CanvasSync
       def perform(chain_definition, globals = {})
         @globals = globals
 
-        if !globals[:updated_after].present? || globals[:updated_after] == true
+        if globals[:updated_after] == false
+          globals[:updated_after] = nil
+        elsif !globals[:updated_after].present? || globals[:updated_after] == true
           last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
           globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
@@ -36,8 +38,10 @@ module CanvasSync
       end
 
       def should_full_sync?(opt)
+        return true unless last_full_sync.present?
         return false unless opt.is_a?(String)
-        case r.strip
+
+        case opt.strip
         when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
           m = Regexp.last_match
           day = m[1]
@@ -62,7 +66,7 @@ module CanvasSync
       end
 
       def last_full_sync
-        last_full_sync_record.started_at
+        last_full_sync_record&.started_at
       end
 
       def genre

data/lib/canvas_sync/jobs/report_starter.rb

@@ -11,6 +11,7 @@ module CanvasSync
       # @return [nil]
       def perform(report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
 
         report_id = start_report(account_id, report_name, report_params)
         # TODO: Restore report caching support (does nayone actually use it?)

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_modules][:report_columns],
           ContextModule,
           mapping[:context_modules][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -18,7 +18,8 @@ module CanvasSync
           report_file_path,
           mapping[options[:mapping].to_sym][:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target
+          conflict_target ? conflict_target.to_sym : conflict_target,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,7 +21,6 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
-
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -75,6 +74,7 @@ module CanvasSync
           mapping[:users][:report_columns],
           User,
           mapping[:users][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -84,6 +84,7 @@ module CanvasSync
           mapping[:pseudonyms][:report_columns],
           Pseudonym,
           mapping[:pseudonyms][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -92,7 +93,8 @@ module CanvasSync
           report_file_path,
           mapping[:accounts][:report_columns],
           Account,
-          mapping[:accounts][:conflict_target].to_sym
+          mapping[:accounts][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -102,6 +104,7 @@ module CanvasSync
           mapping[:courses][:report_columns],
           Course,
           mapping[:courses][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -111,6 +114,7 @@ module CanvasSync
           mapping[:enrollments][:report_columns],
           Enrollment,
           mapping[:enrollments][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -120,6 +124,7 @@ module CanvasSync
           mapping[:sections][:report_columns],
           Section,
           mapping[:sections][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -129,6 +134,7 @@ module CanvasSync
           mapping[:xlist][:report_columns],
           Section,
           mapping[:xlist][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -138,6 +144,7 @@ module CanvasSync
           mapping[:groups][:report_columns],
           Group,
           mapping[:groups][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -148,6 +155,7 @@ module CanvasSync
           mapping[:group_memberships][:report_columns],
           GroupMembership,
           mapping[:group_memberships][:conflict_target].to_sym,
+          import_args: @options
         )
       end
     end

data/lib/canvas_sync/processors/submissions_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class SubmissionsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:submissions][:report_columns],
           Submission,
           mapping[:submissions][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta13".freeze
+  VERSION = "0.17.2".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -42,7 +42,7 @@ RSpec.describe CanvasSync do
                 ]
               }]}
             ]]}
-          ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :d=>4}],
+          ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil, :d=>4}],
         })
       end
 
@@ -61,7 +61,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}]
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}]
           })
         end
       end
@@ -80,7 +80,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -100,7 +100,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -120,7 +120,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -140,7 +140,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -160,7 +160,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -181,7 +181,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -202,7 +202,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -223,7 +223,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end