canvas_sync 0.17.29 → 0.17.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32eaa6fa25c776f85402a9b3fb36e20f69d8954106cca5759db80ee01298b5a5
-  data.tar.gz: 9d2b05d16d6770cbe3b0e4053e5c22c01f416bc61350a14131c509978d9305d5
+  metadata.gz: ac101dc1c7a2d590408677aac25218ac20b762d993413c57ae9f8164b50fb3ad
+  data.tar.gz: c1d4785903974a78e8efc0612e5b36b0c342c6fb8bcfb615616c1e6002cb2186
 SHA512:
-  metadata.gz: 1abd4a0de86b2f22b8f131750533c23507b3b0d354d24a4cd645e5bf368863561e386a2165ac046444fb2b86ca5114ef0de7c7fb45979eb780ea5d01640d9ab9
-  data.tar.gz: a6f794c96ec97e3e39090421d4c339b3afb8d47024332a87b90324b17e2e13c3bc56455c781d0a95c1d391d3b84a08059d534792344b357bee14adccff0109d2
+  metadata.gz: 61f0685f874a9476d8130923b191e7ee1389fcf1cd7374f7b482c11c1e073b30a69796f43b02894cfd37800f7d2240e52d5e48291e8b0f8f8c3698d4f3e84938
+  data.tar.gz: d2a76163cf0a26ba32a96811c1d9e5ac63b6cc7bb23255a27416562e71bd25ee7ae1d57c0d8b21c8043701c9f76eed219469cf48c82883d68d403f938d4f6b0a

data/lib/canvas_sync/job_batches/batch.rb

@@ -28,7 +28,7 @@ module CanvasSync
 
       delegate :redis, to: :class
 
-      BID_EXPIRE_TTL = 2_592_000
+      BID_EXPIRE_TTL = 90.days.to_i
       SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
       BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
@@ -101,6 +101,8 @@ module CanvasSync
               r.hincrby("BID-#{parent_bid}", "children", 1)
               r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
               r.zadd("BID-#{parent_bid}-bids", created_at, bid)
+            else
+              r.zadd("BID-ROOT-bids", created_at, bid)
             end
           end
 
@@ -242,7 +244,7 @@ module CanvasSync
           _, pending, failed, children, complete, parent_bid = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.sadd("BID-#{bid}-failed", jid)
 
               r.hincrby("BID-#{bid}", "pending", 0)
@@ -260,29 +262,19 @@ module CanvasSync
           end
         end
 
+        # Dead jobs are a Sidekiq feature.
+        # If this is called for a job, process_failed_job was also called
         def process_dead_job(bid, jid)
-          _, failed, children, complete, parent_bid = redis do |r|
+          _, dead_count = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.sadd("BID-#{bid}-dead", jid)
-
               r.scard("BID-#{bid}-dead")
-              r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-batches-complete")
-              r.hget("BID-#{bid}", "parent_bid")
-
               r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
             end
           end
 
-          if parent_bid
-            redis.multi do
-              r.sadd("BID-#{parent_bid}-dead", jid)
-              r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
-            end
-          end
-
           enqueue_callbacks(:death, bid)
         end
 
@@ -290,7 +282,7 @@ module CanvasSync
           _, failed, pending, children, complete, success, parent_bid, keep_open = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.srem("BID-#{bid}-failed", jid)
 
               r.scard("BID-#{bid}-failed")
@@ -323,7 +315,7 @@ module CanvasSync
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
 
-            r.multi do
+            r.multi do |r|
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -379,6 +371,7 @@ module CanvasSync
           logger.debug {"Cleaning redis of batch #{bid}"}
           redis do |r|
             r.zrem("batches", bid)
+            r.zrem("BID-ROOT-bids", bid)
             r.unlink(
               "BID-#{bid}",
               "BID-#{bid}-callbacks-complete",
@@ -442,12 +435,8 @@ module CanvasSync
       class RedisProxy
         def multi(*args, &block)
           Batch.redis do |r|
-            if block.arity == 1
-              r.multi(*args) do
-                block.call(r)
-              end
-            else
-              r.multi(*args, &block)
+            r.multi(*args) do |r|
+              block.call(r)
             end
           end
         end

data/lib/canvas_sync/job_batches/callback.rb

@@ -69,7 +69,7 @@ module CanvasSync
                 # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
-                  r.multi do
+                  r.multi do |r|
                     r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
                     r.scard("BID-#{origin_bid}-pending_callbacks")
                     r.hget("BID-#{origin_bid}", "success")
@@ -88,7 +88,7 @@ module CanvasSync
             return unless parent_bid
 
             _, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
-              r.multi do
+              r.multi do |r|
                 r.sadd("BID-#{parent_bid}-batches-success", bid)
                 r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
                 r.scard("BID-#{parent_bid}-batches-success")
@@ -116,7 +116,7 @@ module CanvasSync
 
           def complete(bid, status, parent_bid)
             pending, children, success = Batch.redis do |r|
-              r.multi do
+              r.multi do |r|
                 r.hincrby("BID-#{bid}", "pending", 0)
                 r.hincrby("BID-#{bid}", "children", 0)
                 r.scard("BID-#{bid}-batches-success")
@@ -131,7 +131,7 @@ module CanvasSync
 
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               _, _, complete, pending, children, failure = Batch.redis do |r|
-                r.multi do
+                r.multi do |r|
                   r.sadd("BID-#{parent_bid}-batches-complete", bid)
                   r.sadd("BID-#{parent_bid}-batches-failed", bid)
                   r.scard("BID-#{parent_bid}-batches-complete")

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -127,7 +127,7 @@ module CanvasSync
         return @hash_map[bid] if @hash_map.key?(bid)
 
         context_json, editable = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.hget("BID-#{bid}", "context")
             r.hget("BID-#{bid}", "allow_context_changes")
           end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -18,7 +18,7 @@ module CanvasSync
           man_batch_id = SecureRandom.urlsafe_base64(10)
 
           Batch.redis do |r|
-            r.multi do
+            r.multi do |r|
               r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
               r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
               r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
@@ -87,7 +87,7 @@ module CanvasSync
 
       def self.perform_next_sequence_job(man_batch_id)
         root_bid, ordered = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.hget("MNGBID-#{man_batch_id}", "root_bid")
             r.hget("MNGBID-#{man_batch_id}", "ordered")
           end

data/lib/canvas_sync/job_batches/pool.rb

@@ -94,16 +94,16 @@ module CanvasSync
         redis.hincrby(redis_key, "active_count", 0)
       end
 
-      def pending_count
+      def pending_count(r = redis)
         jobs_key = "#{redis_key}-jobs"
         order = self.order || 'fifo'
         case order.to_sym
         when :fifo, :lifo
-          redis.llen(jobs_key)
+          r.llen(jobs_key)
         when :random
-          redis.scard(jobs_key)
+          r.scard(jobs_key)
         when :priority
-          redis.zcard(jobs_key)
+          r.zcard(jobs_key)
         end
       end
 
@@ -114,9 +114,9 @@ module CanvasSync
           # Make sure this is loaded outside of the pipeline
           self.order
 
-          redis.multi do
+          redis.multi do |r|
             r.hincrby(redis_key, "active_count", -1)
-            self.pending_count
+            self.pending_count(r)
           end
         end
 

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def persist_bid_attr(attribute, value)
         if @initialized || @existing
           redis do |r|
-            r.multi do
+            r.multi do |r|
               r.hset(redis_key, attribute, value)
               r.expire(redis_key, Batch::BID_EXPIRE_TTL)
             end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -13,7 +13,7 @@ module CanvasSync::JobBatches::Sidekiq
 
       def drain_zset(key)
         items, _ = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)
           end

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -10,6 +10,8 @@ require_relative "web/helpers"
 module CanvasSync::JobBatches::Sidekiq
   module Web
     DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
 
     def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       app.helpers do
@@ -24,7 +26,9 @@ module CanvasSync::JobBatches::Sidekiq
 
       app.get "/batches" do
         @count = (params['count'] || 25).to_i
-        @current_page, @total_size, @batches = page('batches', params['page'], @count)
+
+        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
+        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
         @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
 
         erb(get_template(:batches))

data/lib/canvas_sync/jobs/sync_submissions_job.rb

@@ -10,7 +10,7 @@ module CanvasSync
       def perform(options)
         super(
           "proserv_student_submissions_csv",
-          merge_report_params(options, { include_all: options[:include_all] }, {}),
+          merge_report_params(options, { include_all: options[:include_all], enrollment_batch_size: options[:enrollment_batch_size] }, {}),
           CanvasSync::Processors::SubmissionsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -22,7 +22,7 @@ module CanvasSync
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
                 b.description = "TermBatchJob(#{term_id}) Root"
                 b.context = local_context
-                b.on(:success, "#{self.class.to_s}.batch_finished")
+                b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
               end
             end
           else

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.29".freeze
+  VERSION = "0.17.32".freeze
 end

data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe CanvasSync::Jobs::SyncSubmissionsJob do
     context "a term id is in the global_options" do
       it "enqueues a ReportStarter for the proserv_student_submissions_csv for the given term" do
         expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-          .with("self", "proserv_student_submissions_csv", { parameters: { enrollment_term_id: 1, include_all: nil } })
+          .with("self", "proserv_student_submissions_csv", { parameters: { enrollment_term_id: 1, include_all: nil, enrollment_batch_size: nil } })
           .and_return("id" => 1)
 
         expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original