canvas_sync 0.17.0.beta12 → 0.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15dd14f4944009bc0b3415acce0230b3c466deb05d6842dc492f878b56aca4f8
-  data.tar.gz: 7beb66a20e2abbaf5016bf7991606680519d2417fe74e4e65c867ded4fa33e54
+  metadata.gz: e870dc1c517c8a91279f114c7f3c46b108555a350a76749ceca14e517b96e723
+  data.tar.gz: 9b13b29d052e41e917c91afca7fcf693c038e537639be1f66b1c7d0eed635705
 SHA512:
-  metadata.gz: 1adc5820509c6395225375c7cb736c02620f06beb974817c58de7fefefeba026d66e8384bb7f8a802fc3bf6c609bdbcb0ab705c77f055aeac41598be7ccd7c27
-  data.tar.gz: a3f48270de430e38c42301f918248f17de468d243e83f011896cc3b0e5f28fcf4b5f5a49220446c138c7ee720d59e41ced118d7e193aacde6a644ee57f14c555
+  metadata.gz: cff6b6a167ce9eac91487cf9ec546c36544d0156e12b3273c57015cdbf18a279be3018f5221dcc55d90df8c6003a78c5150afe489bd2efec834df4e0700ed327
+  data.tar.gz: 2162d8b3a32d82624ddaacee645ad81e5a0c70afb2a40354e284a9091c225cb92a06f83e291aeab3fbba06a85b99d3ea98621c78f4f1b63f43bba1d9258aec64

data/README.md

@@ -112,6 +112,11 @@ If you pass a date to `updated_after`, this logic will be disabled unless you ex
 - `sunday` - A full sync will run every Sunday
 - `saturday/4` - A full sync will run every fourth Saturday
 
+#### Multiple Sync Chains
+If your app uses multiple Sync Chains, you may run into issues with the automatic `updated_after` and `full_sync_every` logic.
+You can fix this by using custom logic or by setting the `batch_genre` parameter when creating the Job Chain. Chains will only
+use chains of the same genre when computing `updated_after` and `full_sync_every`.
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:

data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb

@@ -1,5 +1,7 @@
 class AddFullSyncToCanvasSyncSyncBatch < CanvasSync::MiscHelper::MigrationClass
   def change
     add_column :canvas_sync_sync_batches, :full_sync, :boolean, default: false
+    add_column :canvas_sync_sync_batches, :batch_genre, :string
+    add_column :canvas_sync_sync_batches, :batch_bid, :string
   end
 end

data/lib/canvas_sync.rb

@@ -42,6 +42,22 @@ module CanvasSync
     xlist
   ].freeze
 
+  SUPPORTED_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
+  DEFAULT_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
   SUPPORTED_LIVE_EVENTS = %w[
     course
     enrollment
@@ -109,11 +125,13 @@ module CanvasSync
     def default_provisioning_report_chain(
       models,
       term_scope: nil,
+      term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
       legacy_support: false,
       account_id: nil,
       updated_after: nil,
       full_sync_every: nil,
-      options: {},
+      batch_genre: nil,
+      options: {}
     ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
@@ -164,6 +182,10 @@ module CanvasSync
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
 
+      (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
+        try_add_model_job.call(mdl)
+      end
+
       ###############################
       # Per-term provisioning jobs
       ###############################
@@ -173,11 +195,9 @@ module CanvasSync
       current_chain << per_term_chain
       current_chain = per_term_chain
 
-      try_add_model_job.call('assignments')
-      try_add_model_job.call('submissions')
-      try_add_model_job.call('assignment_groups')
-      try_add_model_job.call('context_modules')
-      try_add_model_job.call('context_module_items')
+      term_scoped_models.each do |mdl|
+        try_add_model_job.call(mdl)
+      end
 
       current_chain.insert(
         generate_provisioning_jobs(models, options, only_split: ['users'])
@@ -191,6 +211,7 @@ module CanvasSync
         legacy_support: legacy_support,
         updated_after: updated_after,
         full_sync_every: full_sync_every,
+        batch_genre: batch_genre,
       }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -60,10 +60,12 @@ module CanvasSync
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          parent_job, sub_index = matching_jobs[0]
-          chain = self.class.get_chain_parameter(parent_job)
+          relative_job, sub_index = matching_jobs[0]
+          parent_job = find_parent_job(relative_job)
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
+          chain = self.class.get_chain_parameter(parent_job)
+
           if parent_job[:job] != needed_parent_type
             old_job = chain[sub_index]
             parent_job = chain[sub_index] = {
@@ -129,6 +131,24 @@ module CanvasSync
         end
       end
 
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
       class << self
         def _job_type_definitions
           @job_type_definitions ||= {}

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -1,9 +1,13 @@
 module CanvasSync
   module Jobs
     class BeginSyncChainJob < CanvasSync::Job
+      attr_reader :globals
+
       def perform(chain_definition, globals = {})
+        @globals = globals
+
         if !globals[:updated_after].present? || globals[:updated_after] == true
-          last_batch = SyncBatch.where(status: 'completed').last
+          last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
           globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
         end
@@ -15,23 +19,27 @@ module CanvasSync
         sync_batch = SyncBatch.create!(
           started_at: DateTime.now,
           full_sync: globals[:updated_after] == nil,
+          batch_genre: genre,
           status: 'processing',
         )
 
         JobBatches::Batch.new.tap do |b|
-          b.description = "CanvasSync Root Batch"
+          b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
           end
+          sync_batch.update(batch_bid: b.bid)
         end
       end
 
       def should_full_sync?(opt)
+        return true unless last_full_sync.present?
         return false unless opt.is_a?(String)
-        case r.strip
+
+        case opt.strip
         when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
           m = Regexp.last_match
           day = m[1]
@@ -52,11 +60,15 @@ module CanvasSync
       end
 
       def last_full_sync_record
-        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true).last
+        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
       end
 
       def last_full_sync
-        last_full_sync_record.started_at
+        last_full_sync_record&.started_at
+      end
+
+      def genre
+        globals[:batch_genre] || "default"
       end
 
       def self.batch_completed(status, options)

data/lib/canvas_sync/jobs/report_starter.rb

@@ -11,6 +11,7 @@ module CanvasSync
       # @return [nil]
       def perform(report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
 
         report_id = start_report(account_id, report_name, report_params)
         # TODO: Restore report caching support (does nayone actually use it?)

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_modules][:report_columns],
           ContextModule,
           mapping[:context_modules][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -18,7 +18,8 @@ module CanvasSync
           report_file_path,
           mapping[options[:mapping].to_sym][:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target
+          conflict_target ? conflict_target.to_sym : conflict_target,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,7 +21,6 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
-
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -75,6 +74,7 @@ module CanvasSync
           mapping[:users][:report_columns],
           User,
           mapping[:users][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -84,6 +84,7 @@ module CanvasSync
           mapping[:pseudonyms][:report_columns],
           Pseudonym,
           mapping[:pseudonyms][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -92,7 +93,8 @@ module CanvasSync
           report_file_path,
           mapping[:accounts][:report_columns],
           Account,
-          mapping[:accounts][:conflict_target].to_sym
+          mapping[:accounts][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -102,6 +104,7 @@ module CanvasSync
           mapping[:courses][:report_columns],
           Course,
           mapping[:courses][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -111,6 +114,7 @@ module CanvasSync
           mapping[:enrollments][:report_columns],
           Enrollment,
           mapping[:enrollments][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -120,6 +124,7 @@ module CanvasSync
           mapping[:sections][:report_columns],
           Section,
           mapping[:sections][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -129,6 +134,7 @@ module CanvasSync
           mapping[:xlist][:report_columns],
           Section,
           mapping[:xlist][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -138,6 +144,7 @@ module CanvasSync
           mapping[:groups][:report_columns],
           Group,
           mapping[:groups][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -148,6 +155,7 @@ module CanvasSync
           mapping[:group_memberships][:report_columns],
           GroupMembership,
           mapping[:group_memberships][:conflict_target].to_sym,
+          import_args: @options
         )
       end
     end

data/lib/canvas_sync/processors/submissions_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class SubmissionsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:submissions][:report_columns],
           Submission,
           mapping[:submissions][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta12".freeze
+  VERSION = "0.17.1".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -42,7 +42,7 @@ RSpec.describe CanvasSync do
                 ]
               }]}
             ]]}
-          ], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
+          ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil, :d=>4}],
         })
       end
 
@@ -61,7 +61,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}]
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}]
           })
         end
       end
@@ -80,7 +80,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -100,7 +100,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -120,7 +120,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -140,7 +140,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -160,7 +160,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -181,7 +181,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -202,7 +202,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -223,7 +223,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end