canvas_sync 0.23.5 → 0.24.0.beta1

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -1,175 +0,0 @@
-require_relative './base_job'
-
-module CanvasSync::JobBatches
-  class ManagedBatchJob < BaseJob
-    def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, preflight_check: nil, desc_prefix: nil, &blk)
-      desc_prefix ||= ''
-
-      if concurrency == 0 || concurrency == nil || concurrency == true
-        concurrency = sub_jobs.count
-      elsif concurrency == false
-        concurrency = 1
-      end
-
-      root_batch = Batch.new
-      man_batch_id = nil
-
-      if concurrency < sub_jobs.count
-        man_batch_id = SecureRandom.urlsafe_base64(10)
-
-        Batch.redis do |r|
-          r.multi do |r|
-            r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
-            r.hset("MNGBID-#{man_batch_id}", "ordered", ordered ? 1 : 0)
-            r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
-            r.hset("MNGBID-#{man_batch_id}", "preflight_check", preflight_check) if preflight_check.present?
-            r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
-
-            mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
-              j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
-              j = ::ActiveJob::Arguments.serialize([j])
-              JSON.unparse(j)
-            end
-            if ordered
-              r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-            else
-              r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-            end
-            r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
-          end
-        end
-
-        root_batch.allow_context_changes = (concurrency == 1)
-        root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
-
-        desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
-      end
-
-      root_batch.context = context
-
-      blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
-
-      root_batch.description = "#{desc_prefix}#{root_batch.description || 'Root'}"
-
-      root_batch.context["managed_batch_bid"] = man_batch_id if man_batch_id
-
-      if concurrency < sub_jobs.count
-        root_batch.placeholder!
-        concurrency.times do
-          perform_next_sequence_job(man_batch_id, skip_preflight: true)
-        end
-      else
-        root_batch.jobs do
-          sub_jobs.each do |j|
-            ChainBuilder.enqueue_job(j)
-          end
-        end
-      end
-
-      root_batch
-    end
-
-    def perform(sub_jobs, **kwargs)
-      self.class.make_batch(sub_jobs, **kwargs)
-    end
-
-    def self.cleanup_redis(status, options)
-      man_batch_id = options['managed_batch_id']
-      Batch.redis do |r|
-        r.del(
-          "MNGBID-#{man_batch_id}",
-          "MNGBID-#{man_batch_id}-jobs",
-        )
-      end
-    end
-
-    def self.job_succeeded_callback(status, options)
-      man_batch_id = options['managed_batch_id']
-      perform_next_sequence_job(man_batch_id)
-    end
-
-    protected
-
-    def self.perform_next_sequence_job(man_batch_id, skip_preflight: false)
-      root_bid, ordered, preflight_check = Batch.redis do |r|
-        r.multi do |r|
-          r.hget("MNGBID-#{man_batch_id}", "root_bid")
-          r.hget("MNGBID-#{man_batch_id}", "ordered")
-          r.hget("MNGBID-#{man_batch_id}", "preflight_check")
-        end
-      end
-
-      if !skip_preflight && preflight_check.present?
-        if preflight_check.include?(".")
-          clazz, method_name = preflight_check.split('.')
-          clazz = clazz.constantize
-        else
-          clazz = Object
-          method_name = preflight_check
-        end
-        preflight_check = ->(*args) { clazz.send(method_name, *args) }
-      else
-        preflight_check = ->(*args) { true }
-      end
-
-      ordered = CanvasSync::MiscHelper.to_boolean(ordered)
-
-      loop do
-        next_job_json = Batch.redis do |r|
-          if ordered
-            r.lpop("MNGBID-#{man_batch_id}-jobs")
-          else
-            r.spop("MNGBID-#{man_batch_id}-jobs")
-          end
-        end
-
-        break unless next_job_json.present?
-
-        next_job = JSON.parse(next_job_json)
-        next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
-
-        preflight_result = preflight_check.call(next_job)
-        if preflight_result == :abort
-          cleanup_redis(nil, { "managed_batch_id" => man_batch_id })
-          break
-        elsif !preflight_check
-          next
-        end
-
-        Batch.new(root_bid).jobs do
-          Batch.new.tap do |batch|
-            batch.description = "Managed Batch Fiber (#{man_batch_id})"
-            batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
-
-            if next_job[:chain_link].present?
-              # Annotate Batch with chain-step info
-              batch.context["csb:chain_link"] = next_job[:chain_link]
-              batch.on(:complete, "#{ChainBuilder.to_s}.chain_step_complete", chain_link: next_job[:chain_link])
-            end
-
-            batch.jobs do
-              ChainBuilder.enqueue_job(next_job)
-            end
-          end
-        end
-
-        break
-      end
-    end
-
-    class ManagedBatchProxy
-      def initialize(real_batch)
-        @real_batch = real_batch
-      end
-
-      delegate_missing_to :real_batch
-
-      def jobs
-        raise "Managed Batches do not support calling .jobs directly!"
-      end
-
-      private
-      attr_reader :real_batch
-    end
-  end
-end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -1,20 +0,0 @@
-require_relative './base_job'
-
-module CanvasSync::JobBatches
-  class SerialBatchJob < BaseJob
-    def self.make_batch(sub_jobs, **kwargs, &blk)
-      ManagedBatchJob.make_batch(
-        sub_jobs,
-        **kwargs,
-        ordered: true,
-        concurrency: false,
-        desc_prefix: 'SerialBatchJob: ',
-        &blk
-      )
-    end
-
-    def perform(sub_jobs, **kwargs)
-      self.class.make_batch(sub_jobs, **kwargs)
-    end
-  end
-end

data/lib/canvas_sync/job_batches/pool.rb

@@ -1,254 +0,0 @@
-module CanvasSync::JobBatches
-  class Pool
-    include RedisModel
-
-    POOL_REFILL = RedisScript.new(Pathname.new(__FILE__) + "../pool_refill.lua")
-
-    attr_reader :pid
-    redis_attr :description
-    redis_attr :created_at
-    redis_attr :concurrency, :int
-    redis_attr :complete_count, :int
-    redis_attr :order
-    redis_attr :on_failed_job, :symbol
-    redis_attr :clean_when_empty, :bool
-
-    def initialize(pooolid = nil, **kwargs)
-      if pooolid
-        @existing = true
-        @pid = pooolid
-      else
-        @pid = SecureRandom.urlsafe_base64(10)
-        initialize_new(**kwargs)
-      end
-    end
-
-    def self.from_pid(pid)
-      raise "PID must be given" unless pid.present?
-      new(pid)
-    end
-
-    def <<(job_desc)
-      add_job(job_desc)
-    end
-
-    def add_job(job_desc)
-      add_jobs([job_desc])
-    end
-
-    def add_jobs(job_descs, skip_refill: false)
-      job_descs.each do |job_desc|
-        wrapper = Batch.new
-        wrapper.description = "Pool Job Wrapper (PID: #{pid})"
-        checkin_event = (on_failed_job == :wait) ? :success : :complete
-        wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
-        wrapper.placeholder!
-
-        job_desc = job_desc.symbolize_keys
-        job_desc = job_desc.merge!(
-          job: job_desc[:job].to_s,
-          pool_wrapper_batch: wrapper.bid,
-        )
-
-        push_job_to_pool(job_desc)
-      end
-      refill_allotment unless skip_refill
-    end
-
-    def keep_open!(token = SecureRandom.urlsafe_base64(10))
-      if block_given?
-        begin
-          token = keep_open!(token)
-          yield
-        ensure
-          let_close!(token)
-        end
-      else
-        redis.multi do |r|
-          r.sadd("#{redis_key}-holds", token)
-          r.expire("#{redis_key}-holds", Batch::BID_EXPIRE_TTL)
-        end
-        token
-      end
-    end
-
-    def let_close!(token = :unset)
-      if token == :unset # Legacy
-        redis.del("#{redis_key}-holds")
-        redis.hset(redis_key, 'keep_open', 'false')
-      else
-        redis.srem("#{redis_key}-holds", token)
-      end
-
-      cleanup_if_empty
-    end
-
-    def cleanup_redis
-      Batch.logger.debug {"Cleaning redis of pool #{pid}"}
-      redis do |r|
-        r.zrem("pools", pid)
-        r.unlink(
-          "#{redis_key}",
-          "#{redis_key}-jobs",
-        )
-      end
-    end
-
-    def cleanup_if_empty
-      self.order
-
-      activec, pactivec, pendingc, clean_when_empty, keep_open, holds = redis.multi do |r|
-        r.hlen("#{redis_key}-active")
-        r.hget(redis_key, "_active_count")
-        pending_count(r)
-        r.hget(redis_key, 'clean_when_empty')
-        r.hget(redis_key, 'keep_open')
-        r.scard("#{redis_key}-holds")
-      end
-
-      return if keep_open == 'true' || clean_when_empty == 'false' || (holds && holds > 0)
-
-      if activec <= 0 && (pactivec.try(:to_i) || 0) <= 0 && pendingc <= 0
-        cleanup_redis
-      end
-    end
-
-    def active_count(r = redis)
-      r.hlen("#{redis_key}-active") + r.hincrby(redis_key, "_active_count", 0)
-    end
-
-    def active_jobs(r = redis)
-      r.hvals("#{redis_key}-active").map {|desc| JSON.parse(desc)[0] }
-    end
-
-    def pending_count(r = redis)
-      jobs_key = "#{redis_key}-jobs"
-      order = self.order || 'fifo'
-      case order.to_sym
-      when :fifo, :lifo
-        r.llen(jobs_key)
-      when :random
-        r.scard(jobs_key)
-      when :priority
-        r.zcard(jobs_key)
-      end
-    end
-
-    def job_checked_in(status, options)
-      active_count = refill_allotment(status.bid)
-      cleanup_if_empty unless active_count > 0
-    end
-
-    def self.job_checked_in(status, options)
-      pid = options['pool_id']
-      from_pid(pid).job_checked_in(status, options)
-    end
-
-    # Administrative/console method to cleanup expired pools from the WebUI
-    def self.cleanup_redis_index!
-      suffixes = ["", "-active", "-jobs"]
-      r.zrangebyscore("pools", "0", Batch::BID_EXPIRE_TTL.seconds.ago.to_i).each do |pid|
-        r.zrem("pools", pid) if Batch.cleanup_redis_index_for("POOLID-#{pid}", suffixes)
-      end
-    end
-
-    protected
-
-    def redis_key
-      "POOLID-#{pid}"
-    end
-
-    def refill_allotment(checkin_bid = nil)
-      active_count, job_descs = POOL_REFILL.call(redis, [redis_key, "#{redis_key}-jobs", "#{redis_key}-active"], [checkin_bid || ""])
-      return active_count if active_count < 0
-
-      pending_job_descs = job_descs.dup
-
-      added_jobs = {}
-      failed_to_add_jobs = []
-      add_exception = nil
-
-      while pending_job_descs.count > 0
-        begin
-          job_json = pending_job_descs.shift
-          job_desc = ::ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
-
-          wbid = job_desc[:pool_wrapper_batch]
-
-          Batch.new(wbid).jobs do
-            ChainBuilder.enqueue_job(job_desc)
-          end
-
-          added_jobs[wbid] = job_json
-        rescue => ex
-          failed_to_add_jobs << job_json
-          add_exception = ex
-        end
-      end
-
-      redis.multi do |r|
-        r.mapped_hmset("#{redis_key}-active", added_jobs) if added_jobs.count > 0
-        # Release reserved slots now that we've added the jobs to `-active`
-        r.hincrby(redis_key, "_active_count", -job_descs.count)
-
-        r.expire(redis_key, Batch::BID_EXPIRE_TTL)
-        r.expire("#{redis_key}-active", Batch::BID_EXPIRE_TTL)
-        r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
-      end
-
-      # If this happens, we end up in a bad state (as we don't try to re-add items to the pool or refill_allotment again), but
-      #   this should be a _really_ rare case that should only occur if we've lost connection to Redis or something, so we're
-      #   operating on the assumption that if we get here, any recovery logic will fail too
-      if add_exception.present?
-        Batch.logger.error {"Error popping jobs from Pool #{pid}: #{add_exception}"}
-        raise add_exception
-      end
-
-      active_count + added_jobs.count
-    end
-
-    def push_job_to_pool(job_desc)
-      jobs_key = "#{redis_key}-jobs"
-      # This allows duplicate jobs when a Redis Set is used
-      job_desc[:_pool_random_key_] = SecureRandom.urlsafe_base64(10)
-      job_json = JSON.unparse(::ActiveJob::Arguments.serialize([job_desc]))
-      order = self.order
-
-      redis.multi do |r|
-        case order.to_sym
-        when :fifo, :lifo
-          r.rpush(jobs_key, job_json)
-        when :random
-          r.sadd(jobs_key, job_json)
-        when :priority
-          r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
-        end
-        r.expire(redis_key, Batch::BID_EXPIRE_TTL)
-        r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
-      end
-    end
-
-    def self.redis(&blk)
-      Batch.redis &blk
-    end
-    delegate :redis, to: :class
-
-    def flush_pending_attrs
-      super
-      redis.expire(redis_key, Batch::BID_EXPIRE_TTL)
-      redis.zadd("pools", created_at, pid)
-    end
-
-    private
-
-    def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
-      self.created_at = Time.now.utc.to_f
-      self.description = description
-      self.order = order
-      self.concurrency = concurrency
-      self.clean_when_empty = clean_when_empty
-      self.on_failed_job = on_failed_job
-      flush_pending_attrs
-    end
-  end
-end

data/lib/canvas_sync/job_batches/pool_refill.lua

@@ -1,47 +0,0 @@
-
-local poolkey = KEYS[1]
-local qkey = KEYS[2]
-local activekey = KEYS[3]
-
-local checkin_item = ARGV[1]
-
-if redis.call('EXISTS', poolkey) == 0 then
-    return { -1, {} } -- pool doesn't exist
-end
-
-if checkin_item ~= "" then
-    redis.call("HDEL", activekey, checkin_item)
-    redis.call("HINCRBY", poolkey, "complete_count", 1)
-end
-
-local pool_type = redis.call('HGET', poolkey, "order")
-local allotment = tonumber(redis.call("HGET", poolkey, "concurrency"))
-local active = redis.call("HLEN", activekey) + (redis.call("HGET", poolkey, "_active_count") or 0)
-
-local pop_count = allotment - active
-
-local popped_items = {}
-
-if pop_count > 0 then
-    if pool_type == "fifo" then
-        popped_items = redis.call("LPOP", qkey, pop_count) or {}
-    elseif pool_type == "lifo" then
-        popped_items = redis.call("RPOP", qkey, pop_count) or {}
-    elseif pool_type == "random" then
-        popped_items = redis.call("SPOP", qkey, pop_count) or {}
-    elseif pool_type == "priority" then
-        local temp_items = redis.call("ZPOPMAX", qkey, pop_count) or {}
-        for i,v in ipairs(temp_items) do
-            if i % 2 == 1 then
-                table.insert(popped_items, v)
-            end
-        end
-    end
-end
-
--- Reserve slots for these jobs while we return to Ruby and deserialize them
---  This could also be inlined by just storing a key in the queue and storing parameters
---  in a Hash, but this seems more efficient.
-redis.call('HINCRBY', poolkey, "_active_count", #popped_items)
-
-return { active, popped_items }

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -1,67 +0,0 @@
-module CanvasSync::JobBatches
-  module RedisModel
-    extend ActiveSupport::Concern
-
-    class_methods do
-      def redis_attr(key, type = :string, read_only: true)
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{key}=(value)
-            raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
-            @#{key} = value
-            if :#{type} == :json
-              value = JSON.unparse(value)
-            end
-            persist_bid_attr('#{key}', value)
-          end
-
-          def #{key}
-            return @#{key} if defined?(@#{key})
-            if (@initialized || @existing)
-              value = read_bid_attr('#{key}')
-              if :#{type} == :bool
-                value = value == 'true'
-              elsif :#{type} == :int
-                value = value.to_i
-              elsif :#{type} == :float
-                value = value.to_f
-              elsif :#{type} == :json
-                value = JSON.parse(value)
-              elsif :#{type} == :symbol
-                value = value&.to_sym
-              end
-              @#{key} = value
-            end
-          end
-        RUBY
-      end
-    end
-
-    def persist_bid_attr(attribute, value)
-      if @initialized || @existing
-        redis do |r|
-          r.multi do |r|
-            r.hset(redis_key, attribute, value.to_s)
-            r.expire(redis_key, Batch::BID_EXPIRE_TTL)
-          end
-        end
-      else
-        @pending_attrs ||= {}
-        @pending_attrs[attribute] = value.to_s
-      end
-    end
-
-    def read_bid_attr(attribute)
-      redis do |r|
-        r.hget(redis_key, attribute)
-      end
-    end
-
-    def flush_pending_attrs
-      redis do |r|
-        r.mapped_hmset(redis_key, @pending_attrs)
-      end
-      @initialized = true
-      @pending_attrs = {}
-    end
-  end
-end