canvas_sync 0.17.0.beta9 → 0.17.0.beta14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19cb0891aa47a07315b1a27acb63dc16e173afd018ff68a4503259b292928580
-  data.tar.gz: 44394476cf446ee97bcafe471153f53c0a4fbb50c8ce3da8c0a1152201ca4480
+  metadata.gz: 28f1b87e0b26bf68a13a9a1558d8c78746a4058d2fa8a822dc62c5b3b7ad92ea
+  data.tar.gz: 0bbc9ec37d31e848a8ec4997dc91391720deb826b2a93b52528661193788ebe4
 SHA512:
-  metadata.gz: ee2ee669ada624911c67dc22f7aa52e781f03c58904625011a37713b595af7d6348e82be801d22dca1801c623e7316a52fae71f494c727e0b1ce3bd8e31772e0
-  data.tar.gz: d3ef2a3d77d28ba3aa4e343ed051633926c4df7aba2619a798b6ea7e81d9699c5d05e9fa6bfd72d157bde926acacf63e1a8546bc7df66f49093bd89dd057444d
+  metadata.gz: 1cf0b4e64133d94b2841e77399a0dc3a94de42e17832587af14144d9ed28fb1df9953df3486590473abbe9745a8b919c4f910155ff71abe8eb8cc373d0d2704a
+  data.tar.gz: 166edeb5b97e5879b70eebbdf9a3f9848bc91de5a71c5cb6066860b4a6001a272faec24ddf78661d9a5322b3234d1dc69a214bf1e8e8d2c7c16cec6336f01e7a

data/README.md

@@ -103,6 +103,20 @@ It may be one of the following values:
 * An ISO-8601 Date - Will pass the supplied date ad the `updated_after` param for the requested reports
 * `true` (Default) - Will use the start date of the last successful sync
 
+If `updated_after` is true, CanvasSync will, by default, perform a full sync every other Sunday.
+This logic can be customized by passing `full_sync_every` parameter.
+If you pass a date to `updated_after`, this logic will be disabled unless you explicitly pass a `full_sync_every` parameter.
+`full_sync_every` accepts the following format strings:
+- `15%` - Each sync will have a 15% chance of running a full sync
+- `10 days` - A full sync will be run every 10 days
+- `sunday` - A full sync will run every Sunday
+- `saturday/4` - A full sync will run every fourth Saturday
+
+#### Multiple Sync Chains
+If your app uses multiple Sync Chains, you may run into issues with the automatic `updated_after` and `full_sync_every` logic.
+You can fix this by using custom logic or by setting the `batch_genre` parameter when creating the Job Chain. Chains will only
+use chains of the same genre when computing `updated_after` and `full_sync_every`.
+
 ### Extensible chain
 It is sometimes desired to extend or customize the chain of jobs that are run with CanvasSync.
 This can be achieved with the following pattern:

data/db/migrate/20201030210836_add_full_sync_to_canvas_sync_sync_batch.rb

@@ -0,0 +1,7 @@
+class AddFullSyncToCanvasSyncSyncBatch < CanvasSync::MiscHelper::MigrationClass
+  def change
+    add_column :canvas_sync_sync_batches, :full_sync, :boolean, default: false
+    add_column :canvas_sync_sync_batches, :batch_genre, :string
+    add_column :canvas_sync_sync_batches, :batch_bid, :string
+  end
+end

data/lib/canvas_sync.rb

@@ -106,7 +106,16 @@ module CanvasSync
     # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
     #   canvas_sync_client methods require an account ID.
     # @return [Hash]
-    def default_provisioning_report_chain(models, term_scope: nil, legacy_support: false, account_id: nil, updated_after: nil, options: {}) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+    def default_provisioning_report_chain(
+      models,
+      term_scope: nil,
+      legacy_support: false,
+      account_id: nil,
+      updated_after: nil,
+      full_sync_every: nil,
+      batch_genre: nil,
+      options: {}
+    ) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
       return unless models.present?
       models.map! &:to_s
       term_scope = term_scope.to_s if term_scope
@@ -179,7 +188,12 @@ module CanvasSync
       # Wrap it all up
       ###############################
 
-      global_options = { legacy_support: legacy_support, updated_after: updated_after }
+      global_options = {
+        legacy_support: legacy_support,
+        updated_after: updated_after,
+        full_sync_every: full_sync_every,
+        batch_genre: batch_genre,
+      }
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(options[:global]) if options[:global].present?
 

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -28,7 +28,7 @@ module CanvasSync
         database_column_names = mapping.values.map { |value| value[:database_column_name] }
         rows = []
         row_ids = {}
-        database_conflict_column_name = Array(conflict_target).map { |ct| mapping[ct][:database_column_name] }
+        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
 
         CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
           row = yield(row) if block_given?
@@ -67,7 +67,7 @@ module CanvasSync
           condition: condition_sql(klass, columns),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
+        update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
 

data/lib/canvas_sync/job_batches/batch.rb

@@ -371,19 +371,19 @@ module CanvasSync
 
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
-          # redis do |r|
-          #   r.del(
-          #     "BID-#{bid}",
-          #     "BID-#{bid}-callbacks-complete",
-          #     "BID-#{bid}-callbacks-success",
-          #     "BID-#{bid}-failed",
-
-          #     "BID-#{bid}-batches-success",
-          #     "BID-#{bid}-batches-complete",
-          #     "BID-#{bid}-batches-failed",
-          #     "BID-#{bid}-jids",
-          #   )
-          # end
+          redis do |r|
+            r.del(
+              "BID-#{bid}",
+              "BID-#{bid}-callbacks-complete",
+              "BID-#{bid}-callbacks-success",
+              "BID-#{bid}-failed",
+
+              "BID-#{bid}-batches-success",
+              "BID-#{bid}-batches-complete",
+              "BID-#{bid}-batches-failed",
+              "BID-#{bid}-jids",
+            )
+          end
         end
 
         def redis(*args, &blk)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -60,10 +60,12 @@ module CanvasSync
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          parent_job, sub_index = matching_jobs[0]
-          chain = self.class.get_chain_parameter(parent_job)
+          relative_job, sub_index = matching_jobs[0]
+          parent_job = find_parent_job(relative_job)
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
+          chain = self.class.get_chain_parameter(parent_job)
+
           if parent_job[:job] != needed_parent_type
             old_job = chain[sub_index]
             parent_job = chain[sub_index] = {
@@ -129,6 +131,24 @@ module CanvasSync
         end
       end
 
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
       class << self
         def _job_type_definitions
           @job_type_definitions ||= {}

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -1,28 +1,76 @@
 module CanvasSync
   module Jobs
     class BeginSyncChainJob < CanvasSync::Job
+      attr_reader :globals
+
       def perform(chain_definition, globals = {})
+        @globals = globals
+
         if !globals[:updated_after].present? || globals[:updated_after] == true
-          last_batch = SyncBatch.where(status: 'completed').last
+          last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
+          globals[:full_sync_every] ||= "sunday/2"
           globals[:updated_after] = last_batch&.started_at&.iso8601
         end
 
+        if should_full_sync?(globals[:full_sync_every])
+          globals[:updated_after] = nil
+        end
+
         sync_batch = SyncBatch.create!(
           started_at: DateTime.now,
-          status: 'pending',
+          full_sync: globals[:updated_after] == nil,
+          batch_genre: genre,
+          status: 'processing',
         )
 
         JobBatches::Batch.new.tap do |b|
-          b.description = "CanvasSync Root Batch"
+          b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
           end
+          sync_batch.update(batch_bid: b.bid)
+        end
+      end
+
+      def should_full_sync?(opt)
+        return true unless last_full_sync.present?
+        return false unless opt.is_a?(String)
+
+        case r.strip
+        when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
+          m = Regexp.last_match
+          day = m[1]
+          skip = m[2] || "1"
+          Date.new.send(:"#{day}?") && last_full_sync.end_of_day <= (skip.to_i.weeks.ago.end_of_day)
+        when opt.match?(%r{^(\d+)\%$})
+          m = Regexp.last_match
+          rand(100) < m[1].to_i
+        when opt.match?(%r{^(\d+) ?days$})
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.days.ago.end_of_day
+        when opt.match?(%r{^(\d+)$}) # N.days is converted to a string of seconds
+          m = Regexp.last_match
+          last_full_sync.end_of_day <= m[1].to_i.seconds.ago.end_of_day
+        else
+          false
         end
       end
 
+      def last_full_sync_record
+        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
+      end
+
+      def last_full_sync
+        last_full_sync_record&.started_at
+      end
+
+      def genre
+        globals[:batch_genre] || "default"
+      end
+
       def self.batch_completed(status, options)
         sbatch = SyncBatch.find(options['sync_batch_id'])
         sbatch.update!(

data/lib/canvas_sync/jobs/report_starter.rb

@@ -35,7 +35,7 @@ module CanvasSync
       protected
 
       def merge_report_params(options={}, params={}, term_scope: true)
-        term_scope = batch_context[:canvas_term_id] if term_scope == true
+        term_scope = options[:canvas_term_id] || batch_context[:canvas_term_id] if term_scope == true
         if term_scope.present?
           params[:enrollment_term_id] = term_scope
         end
@@ -43,6 +43,7 @@ module CanvasSync
           params[:updated_after] = updated_after
         end
         params.merge!(options[:report_params]) if options[:report_params].present?
+        params.merge!(options[:report_parameters]) if options[:report_parameters].present?
         { parameters: params }
       end
 

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -1,23 +1,8 @@
 module CanvasSync
   module Jobs
     # ActiveJob class that starts a Canvas provisioning report
-    class SyncProvisioningReportJob < CanvasSync::Job
+    class SyncProvisioningReportJob < ReportStarter
       def perform(options)
-        start_report(report_params(options), options)
-      end
-
-      protected
-
-      def start_report(report_params, options)
-        CanvasSync::Jobs::ReportStarter.perform_later(
-          "proservices_provisioning_csv",
-          report_params,
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          options,
-        )
-      end
-
-      def report_params(options, canvas_term_id = options[:canvas_term_id] || batch_context[:canvas_term_id])
         params = {
           include_deleted: true,
         }
@@ -28,11 +13,12 @@ module CanvasSync
           params[model] = true
         end
 
-        params[:enrollment_term_id] = canvas_term_id if canvas_term_id
-
-        params.merge!(options[:report_parameters]) if options[:report_parameters].present?
-
-        { parameters: params }
+        super(
+          "proservices_provisioning_csv",
+          merge_report_params(options, params, {}),
+          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
+          options,
+        )
       end
     end
   end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta9".freeze
+  VERSION = "0.17.0.beta14".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -42,7 +42,7 @@ RSpec.describe CanvasSync do
                 ]
               }]}
             ]]}
-          ], {:legacy_support=>false, :updated_after=>nil, :d=>4}],
+          ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil, :d=>4}],
         })
       end
 
@@ -61,7 +61,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}]
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}]
           })
         end
       end
@@ -80,7 +80,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -100,7 +100,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -120,7 +120,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -140,7 +140,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -160,7 +160,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           })
         end
       end
@@ -181,7 +181,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -202,7 +202,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end
@@ -223,7 +223,7 @@ RSpec.describe CanvasSync do
                   ]
                 }]}
               ]]}
-            ], {:legacy_support=>false, :updated_after=>nil}],
+            ], {:legacy_support=>false, :updated_after=>nil, :full_sync_every=>nil, :batch_genre=>nil}],
           )
         end
       end

data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb

@@ -6,19 +6,18 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
       let!(:term) { FactoryGirl.create(:term) }
 
       it 'enqueues a ReportStarter for a provisioning report for the specified models for each term' do
-        expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
+          expect_any_instance_of(CanvasSync::Jobs::ReportStarter).to receive(:start_report)
           .with(
+            'self',
             'proservices_provisioning_csv',
             {
               parameters: {
                 include_deleted: true,
                 'users' => true,
                 'courses' => true,
-                enrollment_term_id: term.canvas_id
+                enrollment_term_id: term.canvas_id,
               }
             },
-            CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-            { models: ['users', 'courses'], term_scope: 'active' }
           )
 
         set_batch_context(canvas_term_id: term.canvas_id)
@@ -30,8 +29,9 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
 
     context 'a term scope is not specified' do
       it 'enqueues a single ReportStarter for a provisioning report across all terms for the specified models' do
-        expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
+        expect_any_instance_of(CanvasSync::Jobs::ReportStarter).to receive(:start_report)
           .with(
+            'self',
             'proservices_provisioning_csv',
             {
               parameters: {
@@ -40,8 +40,6 @@ RSpec.describe CanvasSync::Jobs::SyncProvisioningReportJob do
                 'courses' => true,
               }
             },
-            CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-            { models: ['users', 'courses'] }
           )
 
         CanvasSync::Jobs::SyncProvisioningReportJob.perform_now(

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_10_18_210836) do
+ActiveRecord::Schema.define(version: 2020_10_30_210836) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -105,6 +105,9 @@ ActiveRecord::Schema.define(version: 2020_10_18_210836) do
     t.string "status"
     t.datetime "created_at"
     t.datetime "updated_at"
+    t.boolean "full_sync", default: false
+    t.string "batch_genre"
+    t.string "batch_bid"
   end
 
   create_table "context_module_items", force: :cascade do |t|