canvas_sync 0.17.0.beta11 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6d329899fbeae89b111e225d044250383eb6f4a0547c0883e0ff555a331f5f3
-  data.tar.gz: ab87348b3d757fbb6acacdf7b7118f7a8da4f290643b34cc7bf0dcdc01f9d109
+  metadata.gz: d8629b92b654035d7c4480345bbf938d385b186c66acd5b4e30f8d76d6dcb109
+  data.tar.gz: 0ef5bae856795745c2f8c8f845533148028076df62f76a395b315ae8b55679ff
 SHA512:
-  metadata.gz: 9090f7b6058736f5b012d1c1bda245a100a9fbe3eb89eb27f27ba29fefcb3f4d36e464315186acf13dbc298b17d6b3d634e7a2ba5fedd086c31fefa94ef70224
-  data.tar.gz: ac4f18e914e1310cc74d209620b97af398c86e22f6c48220e1d3ae15dbb4ab483480ae4ee9e34ad8eb964cdd2bdfb1355dc2cdd142eef59e3d5297f34bae78f8
+  metadata.gz: c08b8308faa3af77ba3828ef122d20e08d2dcf9f5a57e26f2274f000c23fa7d870638e9c2c37256139db6e63d9bc21e7b1497d5b257696c11c224ec9c77ae92e
+  data.tar.gz: 63ddbbaf804dd4e547885c9817c7793646fae10c6f54d76bcf82643cc821d3f72c2a3b76a085e7720c1ec50360cd327f8498481c2d6d04cd829677d0d732259a

data/README.md

@@ -103,6 +103,20 @@ It may be one of the following values:
 * An ISO-8601 Date - Will pass the supplied date ad the `updated_after` param for the requested reports
 * `true` (Default) - Will use the start date of the last successful sync
 
+If `updated_after` is true, CanvasSync will, by default, perform a full sync every other Sunday.
+This logic can be customized by passing `full_sync_every` parameter.
+If you pass a date to `updated_after`, this logic will be disabled unless you explicitly pass a `full_sync_every` parameter.
+`full_sync_every` accepts the following format strings:
+- `15%` - Each sync will have a 15% chance of running a full sync
+- `10 days` - A full sync will be run every 10 days
+- `sunday` - A full sync will run every Sunday
+- `saturday/4` - A full sync will run every fourth Saturday
+
+#### Multiple Sync Chains
+If your app uses multiple Sync Chains, you may run into issues with the automatic `updated_after` and `full_sync_every` logic.
+You can fix this by using custom logic or by setting the `batch_genre` parameter when creating the Job Chain. Chains will only
+use chains of the same genre when computing `updated_after` and `full_sync_every`.
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:

data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb

@@ -0,0 +1,7 @@
+class AddFullSyncToCanvasSyncSyncBatch < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_sync_batches, :full_sync, :boolean, default: false
+    add_column :canvas_sync_sync_batches, :batch_genre, :string
+    add_column :canvas_sync_sync_batches, :batch_bid, :string
+  end
+end

data/lib/canvas_sync.rb

@@ -42,6 +42,22 @@ module CanvasSync
     xlist
   ].freeze
 
+  SUPPORTED_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
+  DEFAULT_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
   SUPPORTED_LIVE_EVENTS = %w[
     course
     enrollment
@@ -106,7 +122,17 @@ module CanvasSync
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
     # @return [Hash]
-    def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+    def default_provisioning_report_chain(
+      models,
+      term_scope: nil,
+      term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
+      legacy_support: false,
+      account_id: nil,
+      updated_after: nil,
+      full_sync_every: nil,
+      batch_genre: nil,
+      options: {}
+    ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -156,6 +182,10 @@ module CanvasSync
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
 
+      (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
+        try_add_model_job.call(mdl)
+      end
+
       ###############################
       # Per-term provisioning jobs
       ###############################
@@ -165,11 +195,9 @@ module CanvasSync
       current_chain << per_term_chain
       current_chain = per_term_chain
 
-      try_add_model_job.call('assignments')
-      try_add_model_job.call('submissions')
-      try_add_model_job.call('assignment_groups')
-      try_add_model_job.call('context_modules')
-      try_add_model_job.call('context_module_items')
+      term_scoped_models.each do |mdl|
+        try_add_model_job.call(mdl)
+      end
 
       current_chain.insert(
         generate_provisioning_jobs(models, options, only_split: ['users'])
@@ -179,7 +207,12 @@ module CanvasSync
       # Wrap it all up
       ###############################
 
-      global_options = { legacy_support: legacy_support, updated_after: updated_after }
+      global_options = {
+        legacy_support: legacy_support,
+        updated_after: updated_after,
+        full_sync_every: full_sync_every,
+        batch_genre: batch_genre,
+      }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -28,7 +28,7 @@ module CanvasSync
         database_column_names = mapping.values.map { |value| value[:database_column_name] }
         rows = []
         row_ids = {}
-        database_conflict_column_name = Array(conflict_target).map { |ct| mapping[ct][:database_column_name] }
+        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
 
         CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
           row = yield(row) if block_given?
@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
+        update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -60,10 +60,12 @@ module CanvasSync
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          parent_job, sub_index = matching_jobs[0]
-          chain = self.class.get_chain_parameter(parent_job)
+          relative_job, sub_index = matching_jobs[0]
+          parent_job = find_parent_job(relative_job)
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
+          chain = self.class.get_chain_parameter(parent_job)
+
           if parent_job[:job] != needed_parent_type
             old_job = chain[sub_index]
             parent_job = chain[sub_index] = {
@@ -129,6 +131,24 @@ module CanvasSync
         end
       end
 
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
       class << self
         def _job_type_definitions
           @job_type_definitions ||= {}

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -1,28 +1,76 @@
 module CanvasSync
   module Jobs
     class BeginSyncChainJob < CanvasSync::Job
+      attr_reader :globals
+
       def perform(chain_definition, globals = {})
+        @globals = globals
+
         if !globals[:updated_after].present? || globals[:updated_after] == true
-          last_batch = SyncBatch.where(status: 'completed').last
+          last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
+          globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
         end
 
+        if should_full_sync?(globals[:full_sync_every])
+          globals[:updated_after] = nil
+        end
+
         sync_batch = SyncBatch.create!(
           started_at: DateTime.now,
-          status: 'pending',
+          full_sync: globals[:updated_after] == nil,
+          batch_genre: genre,
+          status: 'processing',
         )
 
         JobBatches::Batch.new.tap do |b|
-          b.description = "CanvasSync Root Batch"
+          b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
           end
+          sync_batch.update(batch_bid: b.bid)
+        end
+      end
+
+      def should_full_sync?(opt)
+        return true unless last_full_sync.present?
+        return false unless opt.is_a?(String)
+
+        case r.strip
+        when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
+          m = Regexp.last_match
+          day = m[1]
+          skip = m[2] || "1"
+          Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
+        when opt.match?(%r{^(\d+)\%$})
+          m = Regexp.last_match
+          rand(100) < m[1].to_i
+        when opt.match?(%r{^(\d+) ?days$})
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.days.ago.end_of_day
+        when opt.match?(%r{^(\d+)$}) # N.days is converted to a string of seconds
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.seconds.ago.end_of_day
+        else
+          false
         end
       end
 
+      def last_full_sync_record
+        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
+      end
+
+      def last_full_sync
+        last_full_sync_record&.started_at
+      end
+
+      def genre
+        globals[:batch_genre] || "default"
+      end
+
       def self.batch_completed(status, options)
         sbatch = SyncBatch.find(options['sync_batch_id'])
         sbatch.update!(

data/lib/canvas_sync/jobs/report_starter.rb

@@ -11,6 +11,7 @@ module CanvasSync
       # @return [nil]
       def perform(report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || batch_context[:account_id] || "self"
+        options[:sync_start_time] = DateTime.now.utc.iso8601
 
         report_id = start_report(account_id, report_name, report_params)
         # TODO: Restore report caching support (does nayone actually use it?)

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentGroupsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignment_groups][:report_columns],
           AssignmentGroup,
           mapping[:assignment_groups][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class AssignmentsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:assignments][:report_columns],
           Assignment,
           mapping[:assignments][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModuleItemsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_module_items][:report_columns],
           ContextModuleItem,
           mapping[:context_module_items][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -8,15 +8,16 @@ module CanvasSync
     # @param options [Hash]
     class ContextModulesProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
+      def initialize(report_file_path, options)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           mapping[:context_modules][:report_columns],
           ContextModule,
           mapping[:context_modules][:conflict_target].to_sym,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/normal_processor.rb

@@ -18,7 +18,8 @@ module CanvasSync
           report_file_path,
           mapping[options[:mapping].to_sym][:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target
+          conflict_target ? conflict_target.to_sym : conflict_target,
+          import_args: options
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,7 +21,6 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
-
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -75,6 +74,7 @@ module CanvasSync
           mapping[:users][:report_columns],
           User,
           mapping[:users][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -84,6 +84,7 @@ module CanvasSync
           mapping[:pseudonyms][:report_columns],
           Pseudonym,
           mapping[:pseudonyms][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -92,7 +93,8 @@ module CanvasSync
           report_file_path,
           mapping[:accounts][:report_columns],
           Account,
-          mapping[:accounts][:conflict_target].to_sym
+          mapping[:accounts][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -102,6 +104,7 @@ module CanvasSync
           mapping[:courses][:report_columns],
           Course,
           mapping[:courses][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -111,6 +114,7 @@ module CanvasSync
           mapping[:enrollments][:report_columns],
           Enrollment,
           mapping[:enrollments][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -120,6 +124,7 @@ module CanvasSync
           mapping[:sections][:report_columns],
           Section,
           mapping[:sections][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -129,6 +134,7 @@ module CanvasSync
           mapping[:xlist][:report_columns],
           Section,
           mapping[:xlist][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -138,6 +144,7 @@ module CanvasSync
           mapping[:groups][:report_columns],
           Group,
           mapping[:groups][:conflict_target].to_sym,
+          import_args: @options
         )
       end
 
@@ -148,6 +155,7 @@ module CanvasSync
           mapping[:group_memberships][:report_columns],
           GroupMembership,
           mapping[:group_memberships][:conflict_target].to_sym,
+          import_args: @options
         )
       end
     end