canvas_sync 0.17.0.beta10 → 0.17.0.beta15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e44255a358ea745c6d434b9977d97c37c7dd649ae3fdc9679b7e37546a4e014
-  data.tar.gz: cab18a5115b02798a7f25fcd062db9a6c6ef8f5394c018fb332be766444ca460
+  metadata.gz: f242a2d6641d3d51bff9b73eaf2165588405a468a1806183d20e68d7c324d976
+  data.tar.gz: ca2dc6bf47c7c861c42f6550707028d15d1b54469e1a12e3ce7fedead6204102
 SHA512:
-  metadata.gz: 5b06550842f727c1615e7b84daa7f7f66c05ff2a440f4e319f3ac7bf435e9d89da826d805eedfa5fbe5b7c8f16a9aefa276db182139ceffa6a8f70faca1a7156
-  data.tar.gz: 4b53ae2ef1df19049f981d0e585e800d5291dd203acafda0d9d5ed3015341ce0af35e9e41c437271eec29efb9fc059ae53db2e1f8a176c13382b70eeea663deb
+  metadata.gz: 4d25fa9c0d9155780a087ac94b9a72cadddd9805c60760f5bdf719870245b8f1b8c23e14650646565b7a03b6a7983e329049fb1057ccfd990f662ee479322ffd
+  data.tar.gz: f551da63e671e43194c8a2bd2cfe8f95499417d74599d14307f1470ed678a712492f7ad6a70d7208f57a6b3a22e9fb5e5ced209afb292b93b2f0ce88db02d609

data/README.md

@@ -103,6 +103,20 @@ It may be one of the following values:
 * An ISO-8601 Date - Will pass the supplied date ad the `updated_after` param for the requested reports
 * `true` (Default) - Will use the start date of the last successful sync
 
+If `updated_after` is true, CanvasSync will, by default, perform a full sync every other Sunday.
+This logic can be customized by passing `full_sync_every` parameter.
+If you pass a date to `updated_after`, this logic will be disabled unless you explicitly pass a `full_sync_every` parameter.
+`full_sync_every` accepts the following format strings:
+- `15%` - Each sync will have a 15% chance of running a full sync
+- `10 days` - A full sync will be run every 10 days
+- `sunday` - A full sync will run every Sunday
+- `saturday/4` - A full sync will run every fourth Saturday
+
+#### Multiple Sync Chains
+If your app uses multiple Sync Chains, you may run into issues with the automatic `updated_after` and `full_sync_every` logic.
+You can fix this by using custom logic or by setting the `batch_genre` parameter when creating the Job Chain. Chains will only
+use chains of the same genre when computing `updated_after` and `full_sync_every`.
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:

data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb

@@ -0,0 +1,7 @@
+class AddFullSyncToCanvasSyncSyncBatch < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_sync_batches, :full_sync, :boolean, default: false
+    add_column :canvas_sync_sync_batches, :batch_genre, :string
+    add_column :canvas_sync_sync_batches, :batch_bid, :string
+  end
+end

data/lib/canvas_sync.rb

@@ -42,6 +42,22 @@ module CanvasSync
     xlist
   ].freeze
 
+  SUPPORTED_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
+  DEFAULT_TERM_SCOPE_MODELS = %w[
+    assignments
+    submissions
+    assignment_groups
+    context_modules
+    context_module_items
+  ].freeze
+
   SUPPORTED_LIVE_EVENTS = %w[
     course
     enrollment
@@ -106,7 +122,17 @@ module CanvasSync
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
     # @return [Hash]
-    def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+    def default_provisioning_report_chain(
+      models,
+      term_scope: nil,
+      term_scoped_models: DEFAULT_TERM_SCOPE_MODELS,
+      legacy_support: false,
+      account_id: nil,
+      updated_after: nil,
+      full_sync_every: nil,
+      batch_genre: nil,
+      options: {}
+    ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -156,6 +182,10 @@ module CanvasSync
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
 
+      (SUPPORTED_TERM_SCOPE_MODELS - term_scoped_models).each do |mdl|
+        try_add_model_job.call(mdl)
+      end
+
       ###############################
       # Per-term provisioning jobs
       ###############################
@@ -165,11 +195,9 @@ module CanvasSync
       current_chain << per_term_chain
       current_chain = per_term_chain
 
-      try_add_model_job.call('assignments')
-      try_add_model_job.call('submissions')
-      try_add_model_job.call('assignment_groups')
-      try_add_model_job.call('context_modules')
-      try_add_model_job.call('context_module_items')
+      term_scoped_models.each do |mdl|
+        try_add_model_job.call(mdl)
+      end
 
       current_chain.insert(
         generate_provisioning_jobs(models, options, only_split: ['users'])
@@ -179,7 +207,12 @@ module CanvasSync
       # Wrap it all up
       ###############################
 
-      global_options = { legacy_support: legacy_support, updated_after: updated_after }
+      global_options = {
+        legacy_support: legacy_support,
+        updated_after: updated_after,
+        full_sync_every: full_sync_every,
+        batch_genre: batch_genre,
+      }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -28,7 +28,7 @@ module CanvasSync
         database_column_names = mapping.values.map { |value| value[:database_column_name] }
         rows = []
         row_ids = {}
-        database_conflict_column_name = Array(conflict_target).map { |ct| mapping[ct][:database_column_name] }
+        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
 
         CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
           row = yield(row) if block_given?
@@ -67,7 +67,7 @@ module CanvasSync
           condition: condition_sql(klass, columns),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
+        update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 

data/lib/canvas_sync/job_batches/batch.rb

@@ -126,8 +126,6 @@ module CanvasSync
 
         job_queue = @ready_to_queue = []
 
-        puts "Beginning Batch #{bid}"
-
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
@@ -373,19 +371,19 @@ module CanvasSync
 
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
-          # redis do |r|
-          #   r.del(
-          #     "BID-#{bid}",
-          #     "BID-#{bid}-callbacks-complete",
-          #     "BID-#{bid}-callbacks-success",
-          #     "BID-#{bid}-failed",
-
-          #     "BID-#{bid}-batches-success",
-          #     "BID-#{bid}-batches-complete",
-          #     "BID-#{bid}-batches-failed",
-          #     "BID-#{bid}-jids",
-          #   )
-          # end
+          redis do |r|
+            r.del(
+              "BID-#{bid}",
+              "BID-#{bid}-callbacks-complete",
+              "BID-#{bid}-callbacks-success",
+              "BID-#{bid}-failed",
+
+              "BID-#{bid}-batches-success",
+              "BID-#{bid}-batches-complete",
+              "BID-#{bid}-batches-failed",
+              "BID-#{bid}-jids",
+            )
+          end
         end
 
         def redis(*args, &blk)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -60,10 +60,12 @@ module CanvasSync
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          parent_job, sub_index = matching_jobs[0]
-          chain = self.class.get_chain_parameter(parent_job)
+          relative_job, sub_index = matching_jobs[0]
+          parent_job = find_parent_job(relative_job)
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
+          chain = self.class.get_chain_parameter(parent_job)
+
           if parent_job[:job] != needed_parent_type
             old_job = chain[sub_index]
             parent_job = chain[sub_index] = {
@@ -129,6 +131,24 @@ module CanvasSync
         end
       end
 
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
       class << self
         def _job_type_definitions
           @job_type_definitions ||= {}

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -1,28 +1,76 @@
 module CanvasSync
   module Jobs
     class BeginSyncChainJob < CanvasSync::Job
+      attr_reader :globals
+
       def perform(chain_definition, globals = {})
+        @globals = globals
+
         if !globals[:updated_after].present? || globals[:updated_after] == true
-          last_batch = SyncBatch.where(status: 'completed').last
+          last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
+          globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
         end
 
+        if should_full_sync?(globals[:full_sync_every])
+          globals[:updated_after] = nil
+        end
+
         sync_batch = SyncBatch.create!(
           started_at: DateTime.now,
-          status: 'pending',
+          full_sync: globals[:updated_after] == nil,
+          batch_genre: genre,
+          status: 'processing',
         )
 
         JobBatches::Batch.new.tap do |b|
-          b.description = "CanvasSync Root Batch"
+          b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
           end
+          sync_batch.update(batch_bid: b.bid)
+        end
+      end
+
+      def should_full_sync?(opt)
+        return true unless last_full_sync.present?
+        return false unless opt.is_a?(String)
+
+        case r.strip
+        when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
+          m = Regexp.last_match
+          day = m[1]
+          skip = m[2] || "1"
+          Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
+        when opt.match?(%r{^(\d+)\%$})
+          m = Regexp.last_match
+          rand(100) < m[1].to_i
+        when opt.match?(%r{^(\d+) ?days$})
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.days.ago.end_of_day
+        when opt.match?(%r{^(\d+)$}) # N.days is converted to a string of seconds
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.seconds.ago.end_of_day
+        else
+          false
         end
       end
 
+      def last_full_sync_record
+        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
+      end
+
+      def last_full_sync
+        last_full_sync_record&.started_at
+      end
+
+      def genre
+        globals[:batch_genre] || "default"
+      end
+
       def self.batch_completed(status, options)
         sbatch = SyncBatch.find(options['sync_batch_id'])
         sbatch.update!(

data/lib/canvas_sync/jobs/report_starter.rb

@@ -35,7 +35,7 @@ module CanvasSync
       protected
 
       def merge_report_params(options={}, params={}, term_scope: true)
-        term_scope = batch_context[:canvas_term_id] if term_scope == true
+        term_scope = options[:canvas_term_id] || batch_context[:canvas_term_id] if term_scope == true
         if term_scope.present?
           params[:enrollment_term_id] = term_scope
         end
@@ -43,6 +43,7 @@ module CanvasSync
           params[:updated_after] = updated_after
         end
         params.merge!(options[:report_params]) if options[:report_params].present?
+        params.merge!(options[:report_parameters]) if options[:report_parameters].present?
         { parameters: params }
       end
 

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -1,23 +1,8 @@
 module CanvasSync
   module Jobs
     # ActiveJob class that starts a Canvas provisioning report
-    class SyncProvisioningReportJob < CanvasSync::Job
+    class SyncProvisioningReportJob < ReportStarter
       def perform(options)
-        start_report(report_params(options), options)
-      end
-
-      protected
-
-      def start_report(report_params, options)
-        CanvasSync::Jobs::ReportStarter.perform_later(
-          "proservices_provisioning_csv",
-          report_params,
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          options,
-        )
-      end
-
-      def report_params(options, canvas_term_id = options[:canvas_term_id] || batch_context[:canvas_term_id])
         params = {
           include_deleted: true,
         }
@@ -28,11 +13,12 @@ module CanvasSync
           params[model] = true
         end
 
-        params[:enrollment_term_id] = canvas_term_id if canvas_term_id
-
-        params.merge!(options[:report_parameters]) if options[:report_parameters].present?
-
-        { parameters: params }
+        super(
+          "proservices_provisioning_csv",
+          merge_report_params(options, params, {}),
+          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
+          options,
+        )
       end
     end
   end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta10".freeze
+  VERSION = "0.17.0.beta15".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -42,7 +42,7 @@ RSpec.describe CanvasSync do
                 ]
               }]}
             ]]}
-          ], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
+          ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil, :d=>4}],
         })
       end
 
@@ -61,7 +61,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}]
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}]
           })
         end
       end
@@ -80,7 +80,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -100,7 +100,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -120,7 +120,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -140,7 +140,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -160,7 +160,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -181,7 +181,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -202,7 +202,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -223,7 +223,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end