canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/job_batches/pool.rb

@@ -0,0 +1,245 @@
+module CanvasSync
+  module JobBatches
+    class Pool
+      include RedisModel
+
+      POOL_REFILL = RedisScript.new(Pathname.new(__FILE__) + "../pool_refill.lua")
+
+      attr_reader :pid
+      redis_attr :description
+      redis_attr :created_at
+      redis_attr :concurrency, :int
+      redis_attr :complete_count, :int
+      redis_attr :order
+      redis_attr :on_failed_job, :symbol
+      redis_attr :clean_when_empty, :bool
+
+      def initialize(pooolid = nil, **kwargs)
+        if pooolid
+          @existing = true
+          @pid = pooolid
+        else
+          @pid = SecureRandom.urlsafe_base64(10)
+          initialize_new(**kwargs)
+        end
+      end
+
+      def self.from_pid(pid)
+        raise "PID must be given" unless pid.present?
+        new(pid)
+      end
+
+      def <<(job_desc)
+        add_job(job_desc)
+      end
+
+      def add_job(job_desc)
+        add_jobs([job_desc])
+      end
+
+      def add_jobs(job_descs, skip_refill: false)
+        job_descs.each do |job_desc|
+          wrapper = Batch.new
+          wrapper.description = "Pool Job Wrapper (PID: #{pid})"
+          checkin_event = (on_failed_job == :wait) ? :success : :complete
+          wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
+          wrapper.jobs {}
+
+          job_desc = job_desc.symbolize_keys
+          job_desc = job_desc.merge!(
+            job: job_desc[:job].to_s,
+            pool_wrapper_batch: wrapper.bid,
+          )
+
+          push_job_to_pool(job_desc)
+        end
+        refill_allotment unless skip_refill
+      end
+
+      def keep_open!
+        if block_given?
+          begin
+            keep_open!
+            yield
+          ensure
+            let_close!
+          end
+        else
+          redis.hset(redis_key, 'keep_open', 'true')
+        end
+      end
+
+      def let_close!
+        redis.hset(redis_key, 'keep_open', 'false')
+        cleanup_if_empty
+      end
+
+      def cleanup_redis
+        Batch.logger.debug {"Cleaning redis of pool #{pid}"}
+        redis do |r|
+          r.zrem("pools", pid)
+          r.unlink(
+            "#{redis_key}",
+            "#{redis_key}-jobs",
+          )
+        end
+      end
+
+      def cleanup_if_empty
+        self.order
+
+        activec, pactivec, pendingc, clean_when_empty, keep_open = redis.multi do |r|
+          r.hlen("#{redis_key}-active")
+          r.hget(redis_key, "_active_count")
+          pending_count(r)
+          r.hget(redis_key, 'clean_when_empty')
+          r.hget(redis_key, 'keep_open')
+        end
+
+        return if keep_open == 'true' || clean_when_empty == 'false'
+
+        if activec <= 0 && (pactivec.try(:to_i) || 0) <= 0 && pendingc <= 0
+          cleanup_redis
+        end
+      end
+
+      def active_count(r = redis)
+        r.hlen("#{redis_key}-active") + r.hincrby(redis_key, "_active_count", 0)
+      end
+
+      def active_jobs(r = redis)
+        r.hvals("#{redis_key}-active").map {|desc| JSON.parse(desc)[0] }
+      end
+
+      def pending_count(r = redis)
+        jobs_key = "#{redis_key}-jobs"
+        order = self.order || 'fifo'
+        case order.to_sym
+        when :fifo, :lifo
+          r.llen(jobs_key)
+        when :random
+          r.scard(jobs_key)
+        when :priority
+          r.zcard(jobs_key)
+        end
+      end
+
+      def job_checked_in(status, options)
+        active_count = refill_allotment(status.bid)
+        cleanup_if_empty unless active_count > 0
+      end
+
+      def self.job_checked_in(status, options)
+        pid = options['pool_id']
+        from_pid(pid).job_checked_in(status, options)
+      end
+
+      # Administrative/console method to cleanup expired pools from the WebUI
+      def self.cleanup_redis_index!
+        suffixes = ["", "-active", "-jobs"]
+        r.zrangebyscore("pools", "0", Batch::BID_EXPIRE_TTL.seconds.ago.to_i).each do |pid|
+          r.zrem("pools", pid) if Batch.cleanup_redis_index_for("POOLID-#{pid}", suffixes)
+        end
+      end
+
+      protected
+
+      def redis_key
+        "POOLID-#{pid}"
+      end
+
+      def refill_allotment(checkin_bid = nil)
+        active_count, job_descs = POOL_REFILL.call(redis, [redis_key, "#{redis_key}-jobs", "#{redis_key}-active"], [checkin_bid || ""])
+        return active_count if active_count < 0
+
+        pending_job_descs = job_descs.dup
+
+        added_jobs = {}
+        failed_to_add_jobs = []
+        add_exception = nil
+
+        while pending_job_descs.count > 0
+          begin
+            job_json = pending_job_descs.shift
+            job_desc = ::ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
+
+            wbid = job_desc[:pool_wrapper_batch]
+
+            Batch.new(wbid).jobs do
+              ChainBuilder.enqueue_job(job_desc)
+            end
+
+            added_jobs[wbid] = job_json
+          rescue => ex
+            failed_to_add_jobs << job_json
+            add_exception = ex
+          end
+        end
+
+        redis.multi do |r|
+          r.mapped_hmset("#{redis_key}-active", added_jobs) if added_jobs.count > 0
+          # Release reserved slots now that we've added the jobs to `-active`
+          r.hincrby(redis_key, "_active_count", -job_descs.count)
+
+          r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+          r.expire("#{redis_key}-active", Batch::BID_EXPIRE_TTL)
+          r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
+        end
+
+        # If this happens, we end up in a bad state (as we don't try to re-add items to the pool or refill_allotment again), but
+        #   this should be a _really_ rare case that should only occur if we've lost connection to Redis or something, so we're
+        #   operating on the assumption that if we get here, any recovery logic will fail too
+        if add_exception.present?
+          Batch.logger.error {"Error popping jobs from Pool #{pid}: #{add_exception}"}
+          raise add_exception
+        end
+
+        active_count + added_jobs.count
+      end
+
+      def push_job_to_pool(job_desc)
+        jobs_key = "#{redis_key}-jobs"
+        # This allows duplicate jobs when a Redis Set is used
+        job_desc[:_pool_random_key_] = SecureRandom.urlsafe_base64(10)
+        job_json = JSON.unparse(::ActiveJob::Arguments.serialize([job_desc]))
+        order = self.order
+
+        redis.multi do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.rpush(jobs_key, job_json)
+          when :random
+            r.sadd(jobs_key, job_json)
+          when :priority
+            r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
+          end
+          r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+          r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
+        end
+      end
+
+      def self.redis(&blk)
+        Batch.redis &blk
+      end
+      delegate :redis, to: :class
+
+      def flush_pending_attrs
+        super
+        redis.expire(redis_key, Batch::BID_EXPIRE_TTL)
+        redis.zadd("pools", created_at, pid)
+      end
+
+      private
+
+      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
+        self.created_at = Time.now.utc.to_f
+        self.description = description
+        self.order = order
+        self.concurrency = concurrency
+        self.clean_when_empty = clean_when_empty
+        self.on_failed_job = on_failed_job
+        flush_pending_attrs
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/pool_refill.lua

@@ -0,0 +1,47 @@
+
+local poolkey = KEYS[1]
+local qkey = KEYS[2]
+local activekey = KEYS[3]
+
+local checkin_item = ARGV[1]
+
+if redis.call('EXISTS', poolkey) == 0 then
+    return { -1, {} } -- pool doesn't exist
+end
+
+if checkin_item ~= "" then
+    redis.call("HDEL", activekey, checkin_item)
+    redis.call("HINCRBY", poolkey, "complete_count", 1)
+end
+
+local pool_type = redis.call('HGET', poolkey, "order")
+local allotment = tonumber(redis.call("HGET", poolkey, "concurrency"))
+local active = redis.call("HLEN", activekey) + (redis.call("HGET", poolkey, "_active_count") or 0)
+
+local pop_count = allotment - active
+
+local popped_items = {}
+
+if pop_count > 0 then
+    if pool_type == "fifo" then
+        popped_items = redis.call("LPOP", qkey, pop_count) or {}
+    elseif pool_type == "lifo" then
+        popped_items = redis.call("RPOP", qkey, pop_count) or {}
+    elseif pool_type == "random" then
+        popped_items = redis.call("SPOP", qkey, pop_count) or {}
+    elseif pool_type == "priority" then
+        local temp_items = redis.call("ZPOPMAX", qkey, pop_count) or {}
+        for i,v in ipairs(temp_items) do
+            if i % 2 == 1 then
+                table.insert(popped_items, v)
+            end
+        end
+    end
+end
+
+-- Reserve slots for these jobs while we return to Ruby and deserialize them
+--  This could also be inlined by just storing a key in the queue and storing parameters
+--  in a Hash, but this seems more efficient.
+redis.call('HINCRBY', poolkey, "_active_count", #popped_items)
+
+return { active, popped_items }

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -0,0 +1,69 @@
+module CanvasSync
+  module JobBatches
+    module RedisModel
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def redis_attr(key, type = :string, read_only: true)
+          class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def #{key}=(value)
+              raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
+              @#{key} = value
+              if :#{type} == :json
+                value = JSON.unparse(value)
+              end
+              persist_bid_attr('#{key}', value)
+            end
+
+            def #{key}
+              return @#{key} if defined?(@#{key})
+              if (@initialized || @existing)
+                value = read_bid_attr('#{key}')
+                if :#{type} == :bool
+                  value = value == 'true'
+                elsif :#{type} == :int
+                  value = value.to_i
+                elsif :#{type} == :float
+                  value = value.to_f
+                elsif :#{type} == :json
+                  value = JSON.parse(value)
+                elsif :#{type} == :symbol
+                  value = value&.to_sym
+                end
+                @#{key} = value
+              end
+            end
+          RUBY
+        end
+      end
+
+      def persist_bid_attr(attribute, value)
+        if @initialized || @existing
+          redis do |r|
+            r.multi do |r|
+              r.hset(redis_key, attribute, value.to_s)
+              r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+            end
+          end
+        else
+          @pending_attrs ||= {}
+          @pending_attrs[attribute] = value.to_s
+        end
+      end
+
+      def read_bid_attr(attribute)
+        redis do |r|
+          r.hget(redis_key, attribute)
+        end
+      end
+
+      def flush_pending_attrs
+        redis do |r|
+          r.mapped_hmset(redis_key, @pending_attrs)
+        end
+        @initialized = true
+        @pending_attrs = {}
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/redis_script.rb

@@ -0,0 +1,163 @@
+require 'pathname'
+require 'digest/sha1'
+require 'erb'
+
+# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
+
+module CanvasSync
+  module JobBatches
+    # {RedisScript} represents a lua script in the filesystem. It loads the script
+    # from disk and handles talking to redis to execute it. Error handling
+    # is handled by {LuaError}.
+    class RedisScript
+
+      # Loads the script file from disk and calculates its +SHA1+ sum.
+      #
+      # @param file [Pathname] the full path to the indicated file
+      def initialize(file)
+        @file = Pathname.new(file)
+      end
+
+      # Passes the script and supplied arguments to redis for evaulation.
+      # It first attempts to use a script redis has already cached by using
+      # the +EVALSHA+ command, but falls back to providing the full script
+      # text via +EVAL+ if redis has not seen this script before. Future
+      # invocations will then use +EVALSHA+ without erroring.
+      #
+      # @param redis [Redis] the redis connection to run against
+      # @param args [*Objects] the arguments to the script
+      # @return [Object] the value passed back by redis after script execution
+      # @raise [LuaError] if the script failed to compile of encountered a
+      #   runtime error
+      def call(redis, *args)
+        t = Time.now
+        begin
+          redis.evalsha(digest, *args)
+        rescue => e
+          e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
+        end
+      rescue => e
+        if LuaError.intercepts?(e)
+          raise LuaError.new(e, @file, content)
+        else
+          raise
+        end
+      end
+
+      def content
+        @content ||= load_lua(@file)
+      end
+
+      def digest
+        @digest ||= Digest::SHA1.hexdigest content
+      end
+
+      private
+
+      def script_path
+        Rails.root + 'app/redis_lua'
+      end
+
+      def relative_path
+        @path ||= @file.relative_path_from(script_path)
+      end
+
+      def load_lua(file)
+        TemplateContext.new(script_path).template(script_path + file)
+      end
+
+      class TemplateContext
+        def initialize(script_path)
+          @script_path = script_path
+        end
+
+        def template(pathname)
+          @partial_templates ||= {}
+          ERB.new(File.read(pathname)).result binding
+        end
+
+        # helper method to include a lua partial within another lua script
+        #
+        # @param relative_path [String] the relative path to the script from
+        #     `script_path`
+        def include_partial(relative_path)
+          unless @partial_templates.has_key? relative_path
+            @partial_templates[relative_path] = nil
+            template( Pathname.new("#{@script_path}/#{relative_path}") )
+          end
+        end
+      end
+
+      # Reformats errors raised by redis representing failures while executing
+      # a lua script. The default errors have confusing messages and backtraces,
+      # and a type of +RuntimeError+. This class improves the message and
+      # modifies the backtrace to include the lua script itself in a reasonable
+      # way.
+      class LuaError < StandardError
+        PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
+        WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
+        CONTEXT_LINE_NUMBER = 2
+
+        attr_reader :error, :file, :content
+
+        # Is this error one that should be reformatted?
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @return [Boolean] is this an error that should be reformatted?
+        def self.intercepts? error
+          error.message =~ PATTERN
+        end
+
+        # Initialize a new {LuaError} from an existing redis error, adjusting
+        # the message and backtrace in the process.
+        #
+        # @param error [StandardError] the original error raised by redis
+        # @param file [Pathname] full path to the lua file the error ocurred in
+        # @param content [String] lua file content the error ocurred in
+        def initialize error, file, content
+          @error = error
+          @file = file
+          @content = content
+
+          @error.message =~ PATTERN
+          _stage, line_number, message = $1, $2, $3
+          error_context = generate_error_context(content, line_number.to_i)
+
+          super "#{message}\n\n#{error_context}\n\n"
+          set_backtrace generate_backtrace file, line_number
+        end
+
+        private
+
+        def generate_error_context(content, line_number)
+          lines = content.lines.to_a
+          beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
+          ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
+          line_number_width = ending_line_number.to_s.length
+
+          (beginning_line_number..ending_line_number).map do |number|
+            indicator = number == line_number ? '=>' : '  '
+            formatted_number = "%#{line_number_width}d" % number
+            " #{indicator} #{formatted_number}: #{lines[number - 1]}"
+          end.join.chomp
+        end
+
+        def generate_backtrace(file, line_number)
+          pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
+          index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
+          pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
+          pre_wolverine.unshift("#{file}:#{line_number}")
+          pre_wolverine
+        end
+
+        def backtrace_before_entering_wolverine(backtrace)
+          backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
+        end
+
+        def line_from_wolverine(line)
+          line.split(':').first.include?(WOLVERINE_LIB_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -0,0 +1,14 @@
+
+local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
+redis.call('HSET', KEYS[1], ARGV[1], 'true')
+
+if previously_scheduled ~= 'true' then
+    local pcb_key = KEYS[1] .. '-pending_callbacks'
+    redis.call('SADD', pcb_key, ARGV[1] .. '-finalize')
+    if ARGV[2] == 'true' then
+        redis.call('SADD', pcb_key, ARGV[1])
+    end
+    redis.call('EXPIRE', pcb_key, ARGV[3])
+end
+
+return previously_scheduled