canvas_sync 0.17.0.beta3 → 0.17.0.beta8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7f0019fb32e5940971c4845d47a317bd4f451dec546a96a505e3162c8a120d4
-  data.tar.gz: 563d6fb7375f3576f8d2eac14752e14ebedbd210cc63b277c311d0e1c846da7e
+  metadata.gz: 5b9dfa016822ff745b64793d0e25e00a9080ba783af50f126c49271cc9a0d021
+  data.tar.gz: eb7120d981dcf6402afac72c2e3d5785f3da59d20ce151952f43de6d8b829ef7
 SHA512:
-  metadata.gz: ce3550e2f0aa94df6c6b712776d605c16cac568f2a70d49ee668ce5ebdc0978654f01d63619d38bdb1a7befe670518210dbc1e847497fababa4428f779bfd775
-  data.tar.gz: 48a0e195407552a88b46ee93d7e8706585bd4729c618cdb807971a8694405c9e442b361da21472cf9c9d301bbc4dddb0c4c28b7faa5a8f3f4daf1a013195d6f3
+  metadata.gz: 98e434b79e7d96fc64db88a879df7f2b3d6e754bafaa9a8b1adba9f3fb3fad6fd4ca44c5b7b0307155065354738ef4a4f0101907f5b6dbf1d2a96fa74a040802
+  data.tar.gz: 3f051aeb858f550588a67ae83d812e8a08d304b3dbe448221c0ffbeb7a6618a1bdd13f1b7bd619344d66b4dcd0d4a9d69162847fbbc36dfb770509bd84339468

data/lib/canvas_sync/job_batches/batch.rb

@@ -118,40 +118,29 @@ module CanvasSync
           @initialized = true
         end
 
-        @ready_to_queue = []
+        job_queue = @ready_to_queue = []
+
+        puts "Beginning Batch #{@bidkey}"
 
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
           yield
         ensure
+          @ready_to_queue = nil
+          append_jobs(job_queue, parent_bid)
           Thread.current[:batch] = parent
         end
 
-        redis do |r|
-          r.multi do
-            if parent_bid
-              r.hincrby("BID-#{parent_bid}", "children", 1)
-              r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
-              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-            end
-
-            r.hincrby(@bidkey, "pending", @ready_to_queue.size)
-            r.hincrby(@bidkey, "total", @ready_to_queue.size)
-            r.expire(@bidkey, BID_EXPIRE_TTL)
-
-            if @ready_to_queue.size > 0
-              r.sadd(@bidkey + "-jids", @ready_to_queue)
-              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
-            end
-          end
-        end
-
-        @ready_to_queue
+        job_queue
       end
 
       def increment_job_queue(jid)
-        @ready_to_queue << jid
+        if @ready_to_queue
+          @ready_to_queue << jid
+        else
+          append_jobs([jid])
+        end
       end
 
       def invalidate_all
@@ -214,6 +203,27 @@ module CanvasSync
         @pending_attrs = {}
       end
 
+      def append_jobs(jids, parent_bid = self.parent_bid)
+        redis do |r|
+          r.multi do
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.hincrby("BID-#{parent_bid}", "total", jids.size)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            r.hincrby(@bidkey, "pending", jids.size)
+            r.hincrby(@bidkey, "total", jids.size)
+            r.expire(@bidkey, BID_EXPIRE_TTL)
+
+            if jids.size > 0
+              r.sadd(@bidkey + "-jids", jids)
+              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+            end
+          end
+        end
+      end
+
       class << self
         def process_failed_job(bid, jid)
           _, pending, failed, children, complete, parent_bid = redis do |r|
@@ -223,7 +233,7 @@ module CanvasSync
               r.hincrby("BID-#{bid}", "pending", 0)
               r.scard("BID-#{bid}-failed")
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-batches-complete")
               r.hget("BID-#{bid}", "parent_bid")
 
               r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
@@ -253,7 +263,7 @@ module CanvasSync
 
               r.scard("BID-#{bid}-dead")
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-batches-complete")
               r.hget("BID-#{bid}", "parent_bid")
 
               r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
@@ -273,17 +283,18 @@ module CanvasSync
         end
 
         def process_successful_job(bid, jid)
-          failed, pending, children, complete, success, total, parent_bid = redis do |r|
+          _, failed, pending, children, complete, success, total, parent_bid = redis do |r|
             r.multi do
+              r.srem("BID-#{bid}-failed", jid)
+
               r.scard("BID-#{bid}-failed")
               r.hincrby("BID-#{bid}", "pending", -1)
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
-              r.scard("BID-#{bid}-success")
+              r.scard("BID-#{bid}-batches-complete")
+              r.scard("BID-#{bid}-batches-success")
               r.hget("BID-#{bid}", "total")
               r.hget("BID-#{bid}", "parent_bid")
 
-              r.srem("BID-#{bid}-failed", jid)
               r.srem("BID-#{bid}-jids", jid)
               r.expire("BID-#{bid}", BID_EXPIRE_TTL)
             end
@@ -358,18 +369,19 @@ module CanvasSync
 
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
-          redis do |r|
-            r.del(
-              "BID-#{bid}",
-              "BID-#{bid}-callbacks-complete",
-              "BID-#{bid}-callbacks-success",
-              "BID-#{bid}-failed",
-
-              "BID-#{bid}-success",
-              "BID-#{bid}-complete",
-              "BID-#{bid}-jids",
-            )
-          end
+          # redis do |r|
+          #   r.del(
+          #     "BID-#{bid}",
+          #     "BID-#{bid}-callbacks-complete",
+          #     "BID-#{bid}-callbacks-success",
+          #     "BID-#{bid}-failed",
+
+          #     "BID-#{bid}-batches-success",
+          #     "BID-#{bid}-batches-complete",
+          #     "BID-#{bid}-batches-failed",
+          #     "BID-#{bid}-jids",
+          #   )
+          # end
         end
 
         def redis(*args, &blk)

data/lib/canvas_sync/job_batches/callback.rb

@@ -87,13 +87,16 @@ module CanvasSync
           def success(bid, status, parent_bid)
             return unless parent_bid
 
-            _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
+            _, _, success, _, _, complete, pending, children, failure = Batch.redis do |r|
               r.multi do
-                r.sadd("BID-#{parent_bid}-success", bid)
-                r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
-                r.scard("BID-#{parent_bid}-success")
-                r.sadd("BID-#{parent_bid}-complete", bid)
-                r.scard("BID-#{parent_bid}-complete")
+                r.sadd("BID-#{parent_bid}-batches-success", bid)
+                r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
+                r.scard("BID-#{parent_bid}-batches-success")
+
+                r.srem("BID-#{parent_bid}-batches-failed", bid)
+                r.sadd("BID-#{parent_bid}-batches-complete", bid)
+                r.scard("BID-#{parent_bid}-batches-complete")
+
                 r.hincrby("BID-#{parent_bid}", "pending", 0)
                 r.hincrby("BID-#{parent_bid}", "children", 0)
                 r.scard("BID-#{parent_bid}-failed")
@@ -112,7 +115,7 @@ module CanvasSync
               r.multi do
                 r.hincrby("BID-#{bid}", "pending", 0)
                 r.hincrby("BID-#{bid}", "children", 0)
-                r.scard("BID-#{bid}-success")
+                r.scard("BID-#{bid}-batches-success")
               end
             end
 
@@ -127,10 +130,11 @@ module CanvasSync
               # callback may add more jobs to the parent batch
 
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
-              _, complete, pending, children, failure = Batch.redis do |r|
+              _, _, complete, pending, children, failure = Batch.redis do |r|
                 r.multi do
-                  r.sadd("BID-#{parent_bid}-complete", bid)
-                  r.scard("BID-#{parent_bid}-complete")
+                  r.sadd("BID-#{parent_bid}-batches-complete", bid)
+                  r.sadd("BID-#{parent_bid}-batches-failed", bid)
+                  r.scard("BID-#{parent_bid}-batches-complete")
                   r.hincrby("BID-#{parent_bid}", "pending", 0)
                   r.hincrby("BID-#{parent_bid}", "children", 0)
                   r.scard("BID-#{parent_bid}-failed")
@@ -141,11 +145,6 @@ module CanvasSync
               end
             end
           end
-
-          def cleanup_redis bid, callback_bid=nil
-            Batch.cleanup_redis bid
-            Batch.cleanup_redis callback_bid if callback_bid
-          end
         end
       end
     end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -104,6 +104,16 @@ module CanvasSync
         end
       end
 
+      # Legacy Support
+      def merge_options(job, options)
+        matching_jobs = find_matching_jobs(job)
+
+        matching_jobs.each do |j|
+          j[:options] ||= {}
+          j[:options].deep_merge!(options)
+        end
+      end
+
       private
 
       def find_matching_jobs(search_job, parent_job = self.base_job)

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -5,6 +5,7 @@ module CanvasSync
     class ConcurrentBatchJob < BaseJob
       def perform(sub_jobs, context: nil)
         Batch.new.tap do |b|
+          b.description = "Concurrent Batch Root"
           b.context = context
           b.jobs do
             sub_jobs.each do |j|

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -21,6 +21,7 @@ module CanvasSync
           end
         end
 
+        root_batch.description = "Serial Batch Root (#{serial_id})"
         root_batch.allow_context_changes = true
         root_batch.context = context
         root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
@@ -61,7 +62,8 @@ module CanvasSync
 
         Batch.new(root_bid).jobs do
           Batch.new.tap do |batch|
-            batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
+          batch.description = "Serial Batch Fiber (#{serial_id})"
+          batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
             batch.jobs do
               ChainBuilder.enqueue_job(next_job)
             end

data/lib/canvas_sync/job_batches/status.rb

@@ -40,10 +40,26 @@ module CanvasSync
           'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
         end
 
+        def success?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
+        end
+
         def child_count
           Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
         end
 
+        def completed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
+        end
+
+        def successful_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
+        end
+
+        def failed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
+        end
+
         def data
           {
             bid: bid,
@@ -52,9 +68,13 @@ module CanvasSync
             pending: pending,
             created_at: created_at,
             complete: complete?,
+            success: success?,
             failure_info: failure_info,
             parent_bid: parent_bid,
-            child_count: child_count
+            child_count: child_count,
+            completed_children_count: completed_children_count,
+            successful_children_count: successful_children_count,
+            failed_children_count: failed_children_count,
           }
         end
       end

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -15,6 +15,7 @@ module CanvasSync
         JobBatches::Batch.new.tap do |b|
           b.description = "CanvasSync Root Batch"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
+          b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
@@ -25,7 +26,7 @@ module CanvasSync
       def self.batch_completed(status, options)
         sbatch = SyncBatch.find(options['sync_batch_id'])
         sbatch.update!(
-          status: status.failures.positive? ? 'failed' : 'completed',
+          status: status.success? ? 'completed' : 'failed',
           completed_at: DateTime.now,
         )
       end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta3".freeze
+  VERSION = "0.17.0.beta8".freeze
 end

data/spec/job_batching/batch_spec.rb

@@ -182,12 +182,12 @@ RSpec.describe CanvasSync::JobBatches::Batch do
     context 'complete' do
       before { batch.on(:complete, Object) }
       # before { batch.increment_job_queue(bid) }
-      before { batch.jobs do TestWorker.perform_async end }
-      before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
+      # before { batch.jobs do TestWorker.perform_async end }
+      # before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
 
       it 'tries to call complete callback' do
         expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
-        CanvasSync::JobBatches::Batch.process_successful_job(bid, 'failed-job-id')
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
       end
     end
 
@@ -199,6 +199,15 @@ RSpec.describe CanvasSync::JobBatches::Batch do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
       end
 
+      it 'tries to call success callback after a previous failure' do
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
+
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
+        expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
+        CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
+      end
+
       it 'cleanups redis key' do
         CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
         expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)

data/spec/job_batching/flow_spec.rb

@@ -2,19 +2,16 @@ require 'spec_helper'
 
 class WorkerA < BatchTestJobBase
   def perform
-    puts 'A'
   end
 end
 
 class WorkerB < BatchTestJobBase
   def perform
-    puts 'B'
   end
 end
 
 class WorkerC < BatchTestJobBase
   def perform
-    puts 'C'
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.0.beta3
+  version: 0.17.0.beta8
 platform: ruby
 authors:
 - Nate Collings
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-26 00:00:00.000000000 Z
+date: 2020-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler