canvas_sync 0.17.26.beta1 → 0.17.27.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce544260a5cc94f9193cb99da3bc55e06ecd1078e0ef6a5afca282cc4d39169f
-  data.tar.gz: 30a5bccedebde1cb00aeb31219b1d9f605672fa8ea58cb50af483cf8f98d15fb
+  metadata.gz: 413b8669b85123f239a5917982347c9cb90bd56229c42e11a92ce8bc619171f5
+  data.tar.gz: b99de51cad35d7c894ed54ab8910cbe6d0b594cadab814683396350dbbc4b95a
 SHA512:
-  metadata.gz: e84bde52492c2276650bd662adff677c0aa6a1c511587886d45fc6d328389af242ade72fe872dda2093df9ebb151aa25a4a353559a31d74330849f0a0c93d159
-  data.tar.gz: 3c30eceedad806932b61abc66456973250c7247ba7431406f1c5440b09b05cc3b7174112e34e78392d8f3e9c89dfcd7eba43377a0f8ee30d6320a00a50303163
+  metadata.gz: e78f116e14b49c88fb04d95b33623c99a31c6a09735491361b0c287a2d48fff2bf061710f5715854b8601b7eb0c1aa3de9bfd93d87d08f71c260eac830a08fce
+  data.tar.gz: 361de97175217d570b0e60f4b3737a38d685ea4404d568ad35df281f495f08288284d734a02e2f8c5809c2a81e3863bb3030a217ce5d5bab71ca60d889eff31c

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -65,7 +65,7 @@ module CanvasSync
         }
 
         row_buffer = nil
-        if defined?(User) && klass == User && csv_column_names.include?(:user_id)
+        if defined?(User) && klass == User && csv_column_names.include?('user_id')
           row_buffer = UserRowBuffer.new(&row_buffer_out)
         else
           row_buffer = NullRowBuffer.new(&row_buffer_out)

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -95,6 +95,10 @@ module CanvasSync
         @flattened = flattened.with_indifferent_access
       end
 
+      def to_h
+        flatten
+      end
+
       private
 
       def get_parent_hash(bid)

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -3,16 +3,19 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ConcurrentBatchJob < BaseJob
+      def self.make_batch(sub_jobs, context: nil, &blk)
+        ManagedBatchJob.make_batch(
+          sub_jobs,
+          ordered: false,
+          concurrency: true,
+          context: context,
+          desc_prefix: 'ConcurrentBatchJob',
+          &blk
+        )
+      end
+
       def perform(sub_jobs, context: nil)
-        Batch.new.tap do |b|
-          b.description = "Concurrent Batch Root"
-          b.context = context
-          b.jobs do
-            sub_jobs.each do |j|
-              ChainBuilder.enqueue_job(j)
-            end
-          end
-        end
+        self.class.make_batch(sub_jobs, context: context)
       end
     end
   end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -3,8 +3,8 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ManagedBatchJob < BaseJob
-      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
-        man_batch_id = SecureRandom.urlsafe_base64(10)
+      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
+        desc_prefix ||= ''
 
         if concurrency == 0 || concurrency == nil || concurrency == true
           concurrency = sub_jobs.count
@@ -14,38 +14,60 @@ module CanvasSync
 
         root_batch = Batch.new
 
-        Batch.redis do |r|
-          r.multi do
-            r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
-            r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
-            r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
-            r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
-
-            mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
-              j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
-              j = ActiveJob::Arguments.serialize([j])
-              JSON.unparse(j)
-            end
-            if ordered
-              r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-            else
-              r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+        if concurrency < sub_jobs.count
+          man_batch_id = SecureRandom.urlsafe_base64(10)
+
+          Batch.redis do |r|
+            r.multi do
+              r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
+              r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
+              r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+              r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
+
+              mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
+                j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
+                j = ActiveJob::Arguments.serialize([j])
+                JSON.unparse(j)
+              end
+              if ordered
+                r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              else
+                r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              end
+              r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
             end
-            r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
           end
+
+          root_batch.allow_context_changes = (concurrency == 1)
+          root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
+
+          desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
         end
 
-        root_batch.description = "Managed Batch Root (#{man_batch_id})"
-        root_batch.allow_context_changes = (concurrency == 1)
         root_batch.context = context
-        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
-        root_batch.jobs {}
 
-        concurrency.times do
-          self.class.perform_next_sequence_job(man_batch_id)
+        blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
+
+        root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
+
+        if concurrency < sub_jobs.count
+          root_batch.jobs {}
+          concurrency.times do
+            perform_next_sequence_job(man_batch_id)
+          end
+        else
+          root_batch.jobs do
+            sub_jobs.each do |j|
+              ChainBuilder.enqueue_job(j)
+            end
+          end
         end
       end
 
+      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
+        self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
+      end
+
       def self.cleanup_redis(status, options)
         man_batch_id = options['managed_batch_id']
         Batch.redis do |r|
@@ -94,6 +116,21 @@ module CanvasSync
           end
         end
       end
+
+      class ManagedBatchProxy
+        def initialize(real_batch)
+          @real_batch = real_batch
+        end
+
+        delegate_missing_to :real_batch
+
+        def jobs
+          raise "Managed Batches do not support calling .jobs directly!"
+        end
+
+        private
+        attr_reader :real_batch
+      end
     end
   end
 end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -3,14 +3,20 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class SerialBatchJob < BaseJob
-      def perform(sub_jobs, context: nil)
-        ManagedBatchJob.new.perform(
+      def self.make_batch(sub_jobs, context: nil, &blk)
+        ManagedBatchJob.make_batch(
           sub_jobs,
-          context: context,
           ordered: true,
           concurrency: false,
+          context: context,
+          desc_prefix: 'SerialBatchJob',
+          &blk
         )
       end
+
+      def perform(sub_jobs, context: nil)
+        self.class.make_batch(sub_jobs, context: context)
+      end
     end
   end
 end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
       end
 
       def drain_zset(key)
-        items, _ = Sidekiq.redis do |r|
+        items, _ = Batch.redis do |r|
           r.multi do
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -31,6 +31,8 @@ module CanvasSync
           status: 'processing',
         )
 
+        globals[:batch_genre] = genre
+        globals[:batch_start_time] = sync_batch.started_at.iso8601
         globals[:sync_batch_id] = sync_batch.id
 
         JobBatches::Batch.new.tap do |b|

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -6,15 +6,42 @@ module CanvasSync
           context = options[:context] || {}
           if options[:term_scope]
             Term.send(options[:term_scope]).find_each.map do |term|
-              local_context = context.merge(canvas_term_id: get_term_id(term))
-              JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
+              term_id = get_term_id(term)
+              local_context = context.merge(canvas_term_id: term_id)
+
+              # Override the delta-syncing date if:
+              # 1. the Term hasn't been synced before or
+              # 2. the Term underwent a period of not syncing
+              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
+              if batch_context[:updated_after]
+                if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
+                  local_context[:updated_after] = term_last_sync.presence
+                end
+              end
+
+              JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
+                b.description = "TermBatchJob(#{term.canvas_id}) Root"
+                b.context = local_context
+                b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
+              end
             end
           else
-            JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
+            JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
           end
         end
       end
 
+      def self.batch_finished(status, opts)
+        ctx = JobBatches::Batch.current_context
+        term_id = ctx[:canvas_term_id]
+        CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
+      end
+
+      def self.last_sync_key(term_id)
+        ctx = JobBatches::Batch.current_context
+        "#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
+      end
+
       def get_term_id(term)
         term.try(:canvas_id) || term.canvas_term_id
       end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.26.beta1".freeze
+  VERSION = "0.17.27.beta1".freeze
 end

data/lib/canvas_sync.rb

@@ -320,5 +320,15 @@ module CanvasSync
       return if invalid.empty?
       raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
     end
+
+    def redis(*args, &blk)
+      JobBatches::Batch.redis(*args, &blk)
+    end
+
+    def redis_prefix
+      pfx = "cs"
+      pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
+      pfx
+    end
   end
 end

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -10,6 +10,11 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       }.to change { User.count }.by(2)
     end
 
+    it 'uses a User row with a sis_id' do
+      subject.process('spec/support/fixtures/reports/users.csv', { models: ['users'] }, 1)
+      expect(User.find_by(canvas_id: 2).sis_id).to eq 'sis_id_2'
+    end
+
     it 'processes courses' do
       expect {
         subject.process('spec/support/fixtures/reports/courses.csv', { models: ['courses'] }, 1)