canvas_sync 0.17.23.beta7 → 0.17.26.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ffd1eae28ffc49cdba7673f1648fbf82853a57768697df38d2a100c05c7ab3f
-  data.tar.gz: 4ab9d6f8c7ce264b69fae4da982d16dfedf40e5b1d1f64fbfd40831f16465bdf
+  metadata.gz: ce544260a5cc94f9193cb99da3bc55e06ecd1078e0ef6a5afca282cc4d39169f
+  data.tar.gz: 30a5bccedebde1cb00aeb31219b1d9f605672fa8ea58cb50af483cf8f98d15fb
 SHA512:
-  metadata.gz: 9619fbc749887158498c00761798cd0b10018ce5962262f27c9ff69c0615c66ffe1ebfa004bdc2ca7374308e53598ee81175f2372e2cde69527ce4672c288626
-  data.tar.gz: a9459bd07489e9c68abd692977e6ca1bc01b06d7dd960560f85e7f38bff4240ac8334aff15b91fb7f7a84897c7fcbd828e623d3b098fa34126b1b4e9be255222
+  metadata.gz: e84bde52492c2276650bd662adff677c0aa6a1c511587886d45fc6d328389af242ade72fe872dda2093df9ebb151aa25a4a353559a31d74330849f0a0c93d159
+  data.tar.gz: 3c30eceedad806932b61abc66456973250c7247ba7431406f1c5440b09b05cc3b7174112e34e78392d8f3e9c89dfcd7eba43377a0f8ee30d6320a00a50303163

data/lib/canvas_sync/job_batches/batch.rb

@@ -19,6 +19,8 @@ require_relative "./chain_builder"
 
 module CanvasSync
   module JobBatches
+    CURRENT_BATCH_THREAD_KEY = :job_batches_batch
+
     class Batch
       include RedisModel
 
@@ -76,14 +78,12 @@ module CanvasSync
       def on(event, callback, options = {})
         return unless Callback::VALID_CALLBACKS.include?(event.to_s)
         callback_key = "#{@bidkey}-callbacks-#{event}"
-        redis do |r|
-          r.multi do
-            r.sadd(callback_key, JSON.unparse({
-              callback: callback,
-              opts: options
-            }))
-            r.expire(callback_key, BID_EXPIRE_TTL)
-          end
+        redis.multi do |r|
+          r.sadd(callback_key, JSON.unparse({
+            callback: callback,
+            opts: options
+          }))
+          r.expire(callback_key, BID_EXPIRE_TTL)
         end
       end
 
@@ -91,18 +91,16 @@ module CanvasSync
         raise NoBlockGivenError unless block_given?
 
         if !@existing && !@initialized
-          parent_bid = Thread.current[:batch]&.bid
+          parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
 
-          redis do |r|
-            r.multi do
-              r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
-              r.expire(@bidkey, BID_EXPIRE_TTL)
+          redis.multi do |r|
+            r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+            r.expire(@bidkey, BID_EXPIRE_TTL)
 
-              if parent_bid
-                r.hincrby("BID-#{parent_bid}", "children", 1)
-                r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-                r.zadd("BID-#{parent_bid}-bids", created_at, bid)
-              end
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+              r.zadd("BID-#{parent_bid}-bids", created_at, bid)
             end
           end
 
@@ -115,11 +113,11 @@ module CanvasSync
         end
 
         begin
-          parent = Thread.current[:batch]
-          Thread.current[:batch] = self
+          parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
+          Thread.current[CURRENT_BATCH_THREAD_KEY] = self
           yield
         ensure
-          Thread.current[:batch] = parent
+          Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
         end
 
         nil
@@ -131,15 +129,11 @@ module CanvasSync
       end
 
       def invalidate_all
-        redis do |r|
-          r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
-        end
+        redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
       end
 
       def parent_bid
-        redis do |r|
-          r.hget(@bidkey, "parent_bid")
-        end
+        redis.hget(@bidkey, "parent_bid")
       end
 
       def parent
@@ -149,7 +143,7 @@ module CanvasSync
       end
 
       def valid?(batch = self)
-        valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") }
+        valid = !redis.exists?("invalidated-bid-#{batch.bid}")
         batch.parent ? valid && valid?(batch.parent) : valid
       end
 
@@ -162,23 +156,19 @@ module CanvasSync
             let_close!
           end
         else
-          redis do |r|
-            r.hset(@bidkey, 'keep_open', true)
-          end
+          redis.hset(@bidkey, 'keep_open', true)
         end
       end
 
       def let_close!
-        _, failed, pending, children, complete, success = redis do |r|
-          r.multi do
-            r.hset(@bidkey, 'keep_open', false)
-
-            r.scard("BID-#{bid}-failed")
-            r.hincrby("BID-#{bid}", "pending", 0)
-            r.hincrby("BID-#{bid}", "children", 0)
-            r.scard("BID-#{bid}-batches-complete")
-            r.scard("BID-#{bid}-batches-success")
-          end
+        _, failed, pending, children, complete, success = redis.multi do |r|
+          r.hset(@bidkey, 'keep_open', false)
+
+          r.scard("BID-#{bid}-failed")
+          r.hincrby("BID-#{bid}", "pending", 0)
+          r.hincrby("BID-#{bid}", "children", 0)
+          r.scard("BID-#{bid}-batches-complete")
+          r.scard("BID-#{bid}-batches-success")
         end
 
         all_success = pending.to_i.zero? && children == success
@@ -191,11 +181,11 @@ module CanvasSync
 
       def self.with_batch(batch)
         batch = self.new(batch) if batch.is_a?(String)
-        parent = Thread.current[:batch]
-        Thread.current[:batch] = batch
+        parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
+        Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
         yield
       ensure
-        Thread.current[:batch] = parent
+        Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
       end
 
       # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
@@ -211,18 +201,14 @@ module CanvasSync
 
       def flush_pending_attrs
         super
-        redis do |r|
-          r.zadd("batches", created_at, bid)
-        end
+        redis.zadd("batches", created_at, bid)
       end
 
       private
 
       def assert_batch_is_open
         unless defined?(@closed)
-          redis do |r|
-            @closed = r.hget(@bidkey, 'success') == 'true'
-          end
+          @closed = redis.hget(@bidkey, 'success') == 'true'
         end
         raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
       end
@@ -234,7 +220,7 @@ module CanvasSync
         redis do |r|
           tme = Time.now.utc.to_f
           added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
-          r.multi do
+          r.multi do |r|
             r.hincrby(@bidkey, "pending", added)
             r.hincrby(@bidkey, "job_count", added)
             r.expire(@bidkey, BID_EXPIRE_TTL)
@@ -244,6 +230,14 @@ module CanvasSync
       end
 
       class << self
+        def current
+          Thread.current[CURRENT_BATCH_THREAD_KEY]
+        end
+
+        def current_context
+          current&.context || {}
+        end
+
         def process_failed_job(bid, jid)
           _, pending, failed, children, complete, parent_bid = redis do |r|
             return unless r.exists?("BID-#{bid}")
@@ -283,11 +277,9 @@ module CanvasSync
           end
 
           if parent_bid
-            redis do |r|
-              r.multi do
-                r.sadd("BID-#{parent_bid}-dead", jid)
-                r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
-              end
+            redis.multi do
+              r.sadd("BID-#{parent_bid}-dead", jid)
+              r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
             end
           end
 
@@ -413,8 +405,21 @@ module CanvasSync
           cleanup_redis(bid)
         end
 
-        def redis(*args, &blk)
-          defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
+        def redis(&blk)
+          return RedisProxy.new unless block_given?
+
+          if Thread.current[:job_batches_redis]
+            yield Thread.current[:job_batches_redis]
+          elsif defined?(::Sidekiq)
+            ::Sidekiq.redis do |r|
+              Thread.current[:job_batches_redis] = r
+              yield r
+            ensure
+              Thread.current[:job_batches_redis] = nil
+            end
+          else
+            # TODO
+          end
         end
 
         def logger
@@ -433,6 +438,42 @@ module CanvasSync
           end
         end
       end
+
+      class RedisProxy
+        def multi(*args, &block)
+          Batch.redis do |r|
+            if block.arity == 1
+              r.multi(*args) do
+                block.call(r)
+              end
+            else
+              r.multi(*args, &block)
+            end
+          end
+        end
+
+        def pipelined(*args, &block)
+          Batch.redis do |r|
+            if block.arity == 1
+              r.pipelined(*args) do
+                block.call(r)
+              end
+            else
+              r.pipelined(*args, &block)
+            end
+          end
+        end
+
+        def method_missing(method_name, *arguments, &block)
+          Batch.redis do |r|
+            r.send(method_name, *arguments, &block)
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          super || Redis.method_defined?(method_name)
+        end
+      end
     end
 
     ActiveJob::Base.include BatchAwareJob

data/lib/canvas_sync/job_batches/batch_aware_job.rb

@@ -6,17 +6,17 @@ module CanvasSync
       included do
         around_perform do |job, block|
           if (@bid) # This _must_ be @bid - not just bid
-            prev_batch = Thread.current[:batch]
+            prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
             begin
-              Thread.current[:batch] = Batch.new(@bid)
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
               block.call
-              Thread.current[:batch].save_context_changes
+              Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
               Batch.process_successful_job(@bid, job_id)
             rescue
               Batch.process_failed_job(@bid, job_id)
               raise
             ensure
-              Thread.current[:batch] = prev_batch
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
             end
           else
             block.call
@@ -24,7 +24,7 @@ module CanvasSync
         end
 
         around_enqueue do |job, block|
-          if (batch = Thread.current[:batch])
+          if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
             @bid = batch.bid
             batch.increment_job_queue(job_id) if @bid
           end
@@ -33,11 +33,11 @@ module CanvasSync
       end
 
       def bid
-        @bid || Thread.current[:batch]&.bid
+        @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
       end
 
       def batch
-        Thread.current[:batch]
+        Thread.current[CURRENT_BATCH_THREAD_KEY]
       end
 
       def batch_context

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -37,16 +37,29 @@ module CanvasSync
         insert_at(-1, new_job)
       end
 
-      def insert_at(position, new_jobs)
+      def insert_at(position, new_jobs, *args, &blk)
         chain = self.class.get_chain_parameter(base_job)
+        if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+          new_jobs = build_job_hash(new_jobs, *args, &blk)
+        elsif args.length > 0
+          raise "Unexpected number of arguments"
+        end
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
         chain.insert(position, *new_jobs)
       end
 
-      def insert(new_jobs, **kwargs)
-        invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
-        raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
-        raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+      def insert(new_jobs, *args, **kwargs, &blk)
+        if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+          job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
+          args << job_kwargs if job_kwargs.present?
+          new_jobs = build_job_hash(new_jobs, *args, &blk)
+          kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
+        else
+          invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+          raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+          raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+          raise "Unexpected number of arguments" if args.length > 0
+        end
 
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
 
@@ -90,7 +103,6 @@ module CanvasSync
         raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
         return nil if matching_jobs.count == 0
 
-
         job = matching_jobs[0][0]
         job = self.class.new(job) unless job.is_a?(ChainBuilder)
         job
@@ -116,8 +128,22 @@ module CanvasSync
         end
       end
 
+      def apply_block(&blk)
+        return unless blk.present?
+        instance_exec(&blk)
+      end
+
       private
 
+      def build_job_hash(job, *params, &blk)
+        hsh = {
+          job: job,
+          parameters: params,
+        }
+        self.class.new(hsh).apply_block(&blk) if blk.present?
+        hsh
+      end
+
       def find_matching_jobs(search_job, parent_job = self.base_job)
         return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
 
@@ -150,6 +176,13 @@ module CanvasSync
       end
 
       class << self
+        def build(job, *args, &blk)
+          new(job).tap do |ch|
+            ch[:parameters] = args
+            ch.apply_block(&blk)
+          end
+        end
+
         def _job_type_definitions
           @job_type_definitions ||= {}
         end

data/lib/canvas_sync/job_batches/pool.rb

@@ -64,18 +64,14 @@ module CanvasSync
             let_close!
           end
         else
-          redis do |r|
-            r.hset(redis_key, 'keep_open', true)
-          end
+          redis.hset(redis_key, 'keep_open', true)
         end
       end
 
       def let_close!
-        _, active_count = redis do |r|
-          r.multi do
-            r.hset(redis_key, 'keep_open', false)
-            r.hincrby(redis_key, "active_count", 0)
-          end
+        _, active_count = redis.multi do |r|
+          r.hset(redis_key, 'keep_open', false)
+          r.hincrby(redis_key, "active_count", 0)
         end
 
         if active_count == 0 && pending_count == 0
@@ -95,35 +91,38 @@ module CanvasSync
       end
 
       def active_count
-        redis do |r|
-          r.hincrby(redis_key, "active_count", 0)
-        end
+        redis.hincrby(redis_key, "active_count", 0)
       end
 
       def pending_count
         jobs_key = "#{redis_key}-jobs"
         order = self.order || 'fifo'
-        redis do |r|
-          case order.to_sym
-          when :fifo, :lifo
-            r.llen(jobs_key)
-          when :random
-            r.scard(jobs_key)
-          when :priority
-            r.zcard(jobs_key)
-          end
+        case order.to_sym
+        when :fifo, :lifo
+          redis.llen(jobs_key)
+        when :random
+          redis.scard(jobs_key)
+        when :priority
+          redis.zcard(jobs_key)
         end
       end
 
       def job_checked_in(status, options)
-        active_count = redis do |r|
+        active_count, pending_count = redis do |r|
           return unless r.exists?(redis_key)
-          r.hincrby(redis_key, "active_count", -1)
+
+          # Make sure this is loaded outside of the pipeline
+          self.order
+
+          redis.multi do
+            r.hincrby(redis_key, "active_count", -1)
+            self.pending_count
+          end
         end
 
         added_count = refill_allotment
-        if active_count == 0 && added_count == 0
-          if clean_when_empty && redis {|r| r.hget(redis_key, 'keep_open') } != 'true'
+        if active_count == 0 && added_count == 0 && pending_count == 0
+          if clean_when_empty && redis.hget(redis_key, 'keep_open') != 'true'
             cleanup_redis
           end
         end
@@ -175,18 +174,16 @@ module CanvasSync
         job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
         order = self.order
 
-        redis do |r|
-          r.multi do
-            case order.to_sym
-            when :fifo, :lifo
-              r.rpush(jobs_key, job_json)
-            when :random
-              r.sadd(jobs_key, job_json)
-            when :priority
-              r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
-            end
-            r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
+        redis.multi do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.rpush(jobs_key, job_json)
+          when :random
+            r.sadd(jobs_key, job_json)
+          when :priority
+            r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
           end
+          r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
         end
       end
 
@@ -194,18 +191,15 @@ module CanvasSync
         jobs_key = "#{redis_key}-jobs"
         order = self.order
 
-        job_json = nil
-        redis do |r|
-          job_json = case order.to_sym
-          when :fifo
-            r.lpop(jobs_key)
-          when :lifo
-            r.rpop(jobs_key)
-          when :random
-            r.spop(jobs_key)
-          when :priority
-            r.zpopmax(jobs_key)
-          end
+        job_json = case order.to_sym
+        when :fifo
+          redis.lpop(jobs_key)
+        when :lifo
+          redis.rpop(jobs_key)
+        when :random
+          redis.spop(jobs_key)
+        when :priority
+          redis.zpopmax(jobs_key)
         end
 
         return nil unless job_json.present?
@@ -218,13 +212,9 @@ module CanvasSync
       end
       delegate :redis, to: :class
 
-      protected
-
       def flush_pending_attrs
         super
-        redis do |r|
-          r.zadd("pools", created_at, pid)
-        end
+        redis.zadd("pools", created_at, pid)
       end
 
       private

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -8,11 +8,11 @@ module CanvasSync
     module Sidekiq
       module WorkerExtension
         def bid
-          Thread.current[:batch].bid
+          Thread.current[CURRENT_BATCH_THREAD_KEY].bid
         end
 
         def batch
-          Thread.current[:batch]
+          Thread.current[CURRENT_BATCH_THREAD_KEY]
         end
 
         def batch_context
@@ -42,7 +42,7 @@ module CanvasSync
 
       class ClientMiddleware
         def call(_worker, msg, _queue, _redis_pool = nil)
-          if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
+          if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
             batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
           end
           yield
@@ -57,17 +57,17 @@ module CanvasSync
       class ServerMiddleware
         def call(_worker, msg, _queue)
           if (bid = msg['bid'])
-            prev_batch = Thread.current[:batch]
+            prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
             begin
-              Thread.current[:batch] = Batch.new(bid)
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(bid)
               yield
-              Thread.current[:batch].save_context_changes
+              Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
               Batch.process_successful_job(bid, msg['jid'])
             rescue
               Batch.process_failed_job(bid, msg['jid'])
               raise
             ensure
-              Thread.current[:batch] = prev_batch
+              Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
             end
           else
             yield

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -14,6 +14,12 @@ module CanvasSync
           globals[:updated_after] = last_batch&.started_at&.iso8601
         end
 
+        # Refuse to run syncs of the same genre if there is a running full sync
+        if last_full_sync_record&.status == 'processing' && last_full_sync > 12.hours.ago
+          Rails.logger.warn("Attempted to start a '#{genre}' sync while a full-sync is still processing.")
+          return
+        end
+
         if should_full_sync?(globals[:full_sync_every])
           globals[:updated_after] = nil
         end
@@ -25,6 +31,8 @@ module CanvasSync
           status: 'processing',
         )
 
+        globals[:sync_batch_id] = sync_batch.id
+
         JobBatches::Batch.new.tap do |b|
           b.description = "CanvasSync Root Batch (SyncBatch##{sync_batch.id})"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
@@ -62,7 +70,7 @@ module CanvasSync
       end
 
       def last_full_sync_record
-        @last_full_sync_record ||= SyncBatch.where(status: 'completed', full_sync: true, batch_genre: genre).last
+        @last_full_sync_record ||= SyncBatch.where(status: ['completed', 'processing'], full_sync: true, batch_genre: genre).last
       end
 
       def last_full_sync

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -23,23 +23,7 @@ module CanvasSync
           end
         end
 
-        if (jobs = options[:sub_jobs]).present?
-          context = options[:context] || {}
-          if options[:term_scope]
-            Term.send(options[:term_scope]).find_each.map do |term|
-              local_context = context.merge(canvas_term_id: get_term_id(term))
-              JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
-            end
-          else
-            JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
-          end
-        end
-      end
-
-      protected
-
-      def get_term_id(term)
-        term.try(:canvas_id) || term.canvas_term_id
+        TermBatchesJob.perform_now(options)
       end
     end
   end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -0,0 +1,23 @@
+module CanvasSync
+  module Jobs
+    class TermBatchesJob < CanvasSync::Job
+      def perform(options)
+        if (jobs = options[:sub_jobs]).present?
+          context = options[:context] || {}
+          if options[:term_scope]
+            Term.send(options[:term_scope]).find_each.map do |term|
+              local_context = context.merge(canvas_term_id: get_term_id(term))
+              JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
+            end
+          else
+            JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
+          end
+        end
+      end
+
+      def get_term_id(term)
+        term.try(:canvas_id) || term.canvas_term_id
+      end
+    end
+  end
+end