canvas_sync 0.17.3.beta3 → 0.17.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54feabc8f9154b633a70a11d15bfeebaf54ab9d9eb24b9a636c27844728ca1a3
-  data.tar.gz: 0c8d2692a83ebb37a518f61d11fb2970b3204f3d68c990c721fa23a7365a91b6
+  metadata.gz: 999dfad29e32b30e204cc26e6f36700752f3f3efc06314293c9d86df1b52b774
+  data.tar.gz: ef2c1b967f0451a3f2029d2aaae5dbe698e93a5a0136a52eeefe184dc1009f36
 SHA512:
-  metadata.gz: 50efdad1e463d363463b72fa2fcc5581f877bdeb2c2440d688b10f451321996d95c7acd610d822fcc943d0220ed5520f4963de12341c1adb4238d131c5d8df04
-  data.tar.gz: 51d40bc898a58a0aab65a85d4c1820ca5da72bf22b4c11e0ee631ad2e815dc344dc4824a2ade24e7485ace7ad0d12692d71cc93054afcfc05269ac22a4f7b4c9
+  metadata.gz: cd96c186dced2cb00a77fe8e729094ce8c9a7878f74c854c6e15fe1e143b773682b747065c231cf9babf777ebe752f9bdb24f130b7b2485dab9efae7e4ba1be9
+  data.tar.gz: c7d3f33ccbc78a5f4fc22ab1c834ebf21c2be0863e7ff6cbabb5af5ebb5f9181783d8a18f2a1a015d56193d0a67289e1f77f12dcd5942d0b42e3ef6e96b82317

data/lib/canvas_sync/job_batches/batch.rb

@@ -99,7 +99,7 @@ module CanvasSync
               if parent_bid
                 r.hincrby("BID-#{parent_bid}", "children", 1)
                 r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-                r.sadd("BID-#{parent_bid}-bids", bid)
+                r.zadd("BID-#{parent_bid}-bids", created_at, bid)
               end
             end
           end
@@ -171,6 +171,13 @@ module CanvasSync
         @bidkey
       end
 
+      def flush_pending_attrs
+        super
+        redis do |r|
+          r.zadd("batches", created_at, bid)
+        end
+      end
+
       private
 
       def assert_batch_is_open
@@ -184,15 +191,16 @@ module CanvasSync
 
       def append_jobs(jids)
         jids = jids.uniq
+        return unless jids.size > 0
+
         redis do |r|
+          tme = Time.now.utc.to_f
+          added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
           r.multi do
-            r.hincrby(@bidkey, "pending", jids.size)
+            r.hincrby(@bidkey, "pending", added)
+            r.hincrby(@bidkey, "job_count", added)
             r.expire(@bidkey, BID_EXPIRE_TTL)
-
-            if jids.size > 0
-              r.sadd(@bidkey + "-jids", jids)
-              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
-            end
+            r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
           end
         end
       end
@@ -200,6 +208,8 @@ module CanvasSync
       class << self
         def process_failed_job(bid, jid)
           _, pending, failed, children, complete, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.sadd("BID-#{bid}-failed", jid)
 
@@ -220,6 +230,8 @@ module CanvasSync
 
         def process_dead_job(bid, jid)
           _, failed, children, complete, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.sadd("BID-#{bid}-dead", jid)
 
@@ -246,6 +258,8 @@ module CanvasSync
 
         def process_successful_job(bid, jid)
           _, failed, pending, children, complete, success, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.srem("BID-#{bid}-failed", jid)
 
@@ -257,13 +271,11 @@ module CanvasSync
               r.hget("BID-#{bid}", "parent_bid")
 
               r.hincrby("BID-#{bid}", "successful-jobs", 1)
-              r.srem("BID-#{bid}-jids", jid)
+              r.zrem("BID-#{bid}-jids", jid)
               r.expire("BID-#{bid}", BID_EXPIRE_TTL)
             end
           end
 
-          # TODO - There seems to be an issue where a :complete callback batch will occasionally run its job, but won't enqueue_callbacks()
-
           all_success = pending.to_i.zero? && children == success
           # if complete or successfull call complete callback (the complete callback may then call successful)
           if (pending.to_i == failed.to_i && children == complete) || all_success
@@ -302,6 +314,11 @@ module CanvasSync
           if callback_args.present? && !callback_params.present?
             logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
+            redis do |r|
+              r.sadd("#{batch_key}-pending_callbacks", event)
+              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
+            end
+
             with_batch(parent_bid) do
               cb_batch = self.new
               cb_batch.callback_params = {
@@ -330,7 +347,8 @@ module CanvasSync
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
           redis do |r|
-            r.del(
+            r.zrem("batches", bid)
+            r.unlink(
               "BID-#{bid}",
               "BID-#{bid}-callbacks-complete",
               "BID-#{bid}-callbacks-success",
@@ -341,10 +359,21 @@ module CanvasSync
               "BID-#{bid}-batches-failed",
               "BID-#{bid}-bids",
               "BID-#{bid}-jids",
+              "BID-#{bid}-pending_callbacks",
             )
           end
         end
 
+        def delete_prematurely!(bid)
+          child_bids = redis do |r|
+            r.zrange("BID-#{bid}-bids", 0, -1)
+          end
+          child_bids.each do |cbid|
+            delete_prematurely!(cbid)
+          end
+          cleanup_redis(bid)
+        end
+
         def redis(*args, &blk)
           defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
         end
@@ -353,8 +382,6 @@ module CanvasSync
           defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
         end
 
-        private
-
         def push_callbacks(args, queue)
           Batch::Callback::worker_class.enqueue_all(args, queue)
         end

data/lib/canvas_sync/job_batches/callback.rb

@@ -52,9 +52,6 @@ module CanvasSync
 
         class Finalize
           def dispatch(status, opts)
-            is_callback_batch = opts['origin'].present?
-            has_callback_batch = opts['callback_bid'].present?
-
             bid = opts["bid"]
             event = opts["event"].to_sym
 
@@ -63,12 +60,20 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
-            if event == :success && !has_callback_batch
-              Batch.cleanup_redis(bid)
-            end
-
-            if event == :success && is_callback_batch && opts['origin']['event'].to_sym == :success
-              Batch.cleanup_redis(opts['origin']['for_bid'])
+            if event == :success
+              if opts['origin'].present?
+                origin_bid = opts['origin']['for_bid']
+                _, pending, success_ran = Batch.redis do |r|
+                  r.multi do
+                    r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
+                    r.scard("BID-#{origin_bid}-pending_callbacks")
+                    r.hget("BID-#{origin_bid}", "success")
+                  end
+                end
+                Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
+              elsif (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
+                Batch.cleanup_redis(bid)
+              end
             end
           end
 
@@ -113,13 +118,7 @@ module CanvasSync
               end
             end
 
-            # If the batch was successful run :success callback, which will call the parent's :complete callback (if necessary)
-            # Also, only trigger the success callback if the :complete callback_batch was successful
-            if pending.to_i.zero? && children == success
-              # Batch.enqueue_callbacks(:success, bid)
-
-            # otherwise check for a parent and call its :complete if needed
-            elsif parent_bid
+            if parent_bid && !(pending.to_i.zero? && children == success)
               # if batch was not successfull check and see if its parent is complete
               # if the parent is complete we trigger the complete callback
               # We don't want to run this if the batch was successfull because the success

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -28,6 +28,10 @@ module CanvasSync
         @hash_map[local_bid]
       end
 
+      def own
+        resolve_hash(@bid_stack[-1]) || {}
+      end
+
       def set_local(new_hash)
         @dirty = true
         local.clear.merge!(new_hash)
@@ -106,7 +110,8 @@ module CanvasSync
           @bid_stack[index]
         else
           pbid = Batch.redis do |r|
-            r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
+            callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
+            callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
           end
           @bid_stack.unshift(pbid)
           pbid

data/lib/canvas_sync/job_batches/pool.rb

@@ -24,6 +24,7 @@ module CanvasSync
       end
 
       def self.from_pid(pid)
+        raise "PID must be given" unless pid.present?
         new(pid)
       end
 
@@ -38,7 +39,7 @@ module CanvasSync
       def add_jobs(job_descs)
         job_descs.each do |job_desc|
           wrapper = Batch.new
-          wrapper.description = "Pool Job Wrapper"
+          wrapper.description = "Pool Job Wrapper (PID: #{pid})"
           checkin_event = (on_failed_job == :wait) ? :success : :complete
           wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
           wrapper.jobs {}
@@ -57,19 +58,42 @@ module CanvasSync
       def cleanup_redis
         Batch.logger.debug {"Cleaning redis of pool #{pid}"}
         redis do |r|
-          r.del(
+          r.zrem("pools", pid)
+          r.unlink(
             "#{redis_key}",
             "#{redis_key}-jobs",
           )
         end
       end
 
+      def active_count
+        redis do |r|
+          r.hincrby(redis_key, "active_count", 0)
+        end
+      end
+
+      def pending_count
+        jobs_key = "#{redis_key}-jobs"
+        order = self.order || 'fifo'
+        redis do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.llen(jobs_key)
+          when :random
+            r.scard(jobs_key)
+          when :priority
+            r.zcard(jobs_key)
+          end
+        end
+      end
+
       def job_checked_in(status, options)
         active_count = redis do |r|
+          return unless r.exists?(redis_key)
           r.hincrby(redis_key, "active_count", -1)
         end
-        added_count = refill_allotment
 
+        added_count = refill_allotment
         if active_count == 0 && added_count == 0
           cleanup_redis if clean_when_empty
         end
@@ -159,29 +183,25 @@ module CanvasSync
         ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
       end
 
-      def pending_count
-        order = self.order
-        redis do |r|
-          case order.to_sym
-          when :fifo, :lifo
-            r.llen(jobs_key)
-          when :random
-            r.scard(jobs_key)
-          when :priority
-            r.zcard(jobs_key)
-          end
-        end
-      end
-
       def self.redis(&blk)
         Batch.redis &blk
       end
       delegate :redis, to: :class
 
+      protected
+
+      def flush_pending_attrs
+        super
+        redis do |r|
+          r.zadd("pools", created_at, pid)
+        end
+      end
+
       private
 
-      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
+      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
         self.created_at = Time.now.utc.to_f
+        self.description = description
         self.order = order
         self.concurrency = concurrency
         self.clean_when_empty = clean_when_empty

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -117,3 +117,5 @@ module CanvasSync
     end
   end
 end
+
+require_relative 'sidekiq/web'

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -0,0 +1,114 @@
+
+begin
+  require "sidekiq/web"
+rescue LoadError
+  # client-only usage
+end
+
+require_relative "web/helpers"
+
+module CanvasSync::JobBatches::Sidekiq
+  module Web
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      app.helpers do
+        include Web::Helpers
+      end
+
+      app.get "/batches" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @batches = page('batches', params['page'], @count)
+        @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        erb(get_template(:batches))
+      end
+
+      app.get "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        @count = (params['count'] || 25).to_i
+        @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
+        @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
+
+        @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
+        @jobs = @jobs.map {|jid, score| jid }
+
+        erb(get_template(:batch))
+      end
+
+      app.post "/batches/all" do
+        if params['delete']
+          drain_zset('batches') do |batches|
+            batches.each do |bid|
+              CanvasSync::JobBatches::Batch.cleanup_redis(bid)
+            end
+          end
+        end
+
+        redirect "#{root_path}batches"
+      end
+
+      app.post "/batches/:bid" do
+        @bid = params[:bid]
+        @batch = CanvasSync::JobBatches::Batch.new(@bid)
+
+        if params['delete']
+          CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
+        end
+
+        redirect_with_query("#{root_path}batches")
+      end
+
+      # =============== POOLS =============== #
+
+      app.get "/pools" do
+        @count = (params['count'] || 25).to_i
+        @current_page, @total_size, @pools = page('pools', params['page'], @count)
+        @pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
+
+        erb(get_template(:pools))
+      end
+
+      app.get "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.new(@pid)
+
+        @count = (params['count'] || 25).to_i
+        @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
+        @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
+
+        erb(get_template(:pool))
+      end
+
+      app.post "/pools/all" do
+        if params['delete']
+          drain_zset('pools') do |pools|
+            pools.each do |pid|
+              CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
+            end
+          end
+        end
+
+        redirect "#{root_path}pools"
+      end
+
+      app.post "/pools/:pid" do
+        @pid = params[:pid]
+        @pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
+
+        if params['delete']
+          @pool.cleanup_redis
+        end
+
+        redirect_with_query("#{root_path}pools")
+      end
+    end
+  end
+end
+
+if defined?(::Sidekiq::Web)
+  ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
+  ::Sidekiq::Web.tabs["Batches"] = "batches"
+  ::Sidekiq::Web.tabs["Pools"] = "pools"
+  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
+end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module CanvasSync::JobBatches::Sidekiq
+  module Web
+    module Helpers
+      VIEW_PATH    = File.expand_path("../web/views", __dir__)
+
+      module_function
+
+      def get_template(name)
+        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+      end
+
+      def drain_zset(key)
+        items, _ = Sidekiq.redis do |r|
+          r.multi do
+            r.zrange(key, 0, -1)
+            r.zremrangebyrank(key, 0, -1)
+          end
+        end
+        yield items
+      end
+
+      def safe_relative_time(time)
+        time = parse_time(time)
+        relative_time(time)
+      end
+
+      def parse_time(time)
+        case time
+        when Time
+          time
+        when Integer, Float
+          Time.at(time)
+        else
+          Time.parse(time.to_s)
+        end
+      end
+    end
+  end
+end