inst-jobs 2.2.1 → 2.4.0

data/lib/delayed/backend/redis/bulk_update.lua

@@ -1,50 +0,0 @@
-local action, id_string, flavor, query, now = unpack(ARGV)
-
-local ids = {}
-
-if string.len(flavor) > 0 then
-  if flavor == 'current' then
-    ids = redis.call('ZRANGE', Keys.queue(query), 0, -1)
-  elseif flavor == 'future' then
-    ids = redis.call('ZRANGE', Keys.future_queue(query), 0, -1)
-  elseif flavor == 'strand' then
-    ids = redis.call('LRANGE', Keys.strand(query), 0, -1)
-  elseif flavor == 'tag' then
-    ids = redis.call('SMEMBERS', Keys.tag(query))
-  end
-else
-  -- can't pass an array to redis/lua, so we split the string here
-  for id in string.gmatch(id_string, "([%w-]+)") do
-    if job_exists(id) then
-      table.insert(ids, id)
-    end
-  end
-end
-
-local count = 0
-for idx, job_id in ipairs(ids) do
-  if action == 'hold' then
-    local queue, strand, locked_by = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'strand', 'locked_by'))
-    if not locked_by then
-      count = count + 1
-      remove_from_queues(job_id, queue, strand)
-      redis.call('HMSET', Keys.job(job_id), 'locked_at', now, 'locked_by', 'on hold', 'attempts', 50)
-    end
-  elseif action == 'unhold' then
-    local queue, locked_by = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'locked_by'))
-    if locked_by == 'on hold' then
-      count = count + 1
-      add_to_queues(job_id, queue, now)
-      redis.call('HDEL', Keys.job(job_id), 'locked_at', 'locked_by')
-      redis.call('HMSET', Keys.job(job_id), 'attempts', 0)
-    end
-  elseif action == 'destroy' then
-    local locked_by = redis.call('HGET', Keys.job(job_id), 'locked_by')
-    if not locked_by or locked_by == 'on hold' then
-      count = count + 1
-      destroy_job(job_id, now)
-    end
-  end
-end
-
-return count

data/lib/delayed/backend/redis/destroy_job.lua

@@ -1,2 +0,0 @@
-local job_id, now = unpack(ARGV)
-destroy_job(job_id, now)

data/lib/delayed/backend/redis/enqueue.lua

@@ -1,29 +0,0 @@
-local job_id, queue, strand, now, for_singleton = unpack(ARGV)
-local strand_key = Keys.strand(strand)
-
--- if this is a singleton job, only queue it up if another doesn't exist on the strand
--- otherwise, delete it and return the other job id
-if for_singleton then
-  local job_ids = redis.call('LRANGE', strand_key, 0, 1)
-  local job_to_check = 1
-  if job_exists(job_ids[1]) and redis.call('HGET', Keys.job(job_ids[1]), 'locked_at') then
-    job_to_check = 2
-  end
-
-  local job_to_check_id = job_ids[job_to_check]
-  if job_exists(job_to_check_id) then
-    -- delete the new job, we found a match
-    redis.call('DEL', Keys.job(job_id))
-    return job_to_check_id
-  end
-end
-
--- if this job is in a strand, add it to the strand queue first
--- if it's not at the front of the strand, we won't enqueue it below
-if strand_key then
-  add_to_strand(job_id, strand)
-end
-
-add_to_queues(job_id, queue, now)
-
-return job_id

data/lib/delayed/backend/redis/fail_job.lua

@@ -1,5 +0,0 @@
-local job_id = unpack(ARGV)
-local failed_at, queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'failed_at', 'queue', 'strand'))
-
-remove_from_queues(job_id, queue, strand)
-redis.call('ZADD', Keys.failed_jobs(), failed_at, job_id)

data/lib/delayed/backend/redis/find_available.lua

@@ -1,3 +0,0 @@
-local queue, limit, offset, min_priority, max_priority, now = unpack(ARGV)
-
-return find_available(queue, limit, offset, min_priority, max_priority, now)

data/lib/delayed/backend/redis/functions.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'redis/scripting'
-
-# This module handles loading the Lua functions into Redis and running them
-module Delayed::Backend::Redis
-class Functions < ::Redis::Scripting::Module
-  def initialize(redis)
-    super(redis, File.dirname(__FILE__))
-  end
-
-  def run_script(script, keys, argv)
-    result = nil
-    ms = Benchmark.ms { result = super }
-    line = 'Redis Jobs Timing: %s (%.1fms)' % [script.name, ms]
-    ActiveRecord::Base.logger.debug(line)
-    result
-  end
-
-  def find_available(queue, limit, offset, min_priority, max_priority, now)
-    run(:find_available, [], [queue, limit, offset, min_priority, max_priority, now.utc.to_f])
-  end
-
-  def get_and_lock_next_available(worker_name, queue, min_priority, max_priority, now)
-    attrs = run(:get_and_lock_next_available, [], [queue, min_priority, max_priority, worker_name, now.utc.to_f])
-    Hash[*attrs]
-  end
-
-  def enqueue(job_id, queue, strand, now)
-    run(:enqueue, [], [job_id, queue, strand, now.utc.to_f])
-  end
-
-  def create_singleton(job_id, queue, strand, now)
-    run(:enqueue, [], [job_id, queue, strand, now.utc.to_f, true])
-  end
-
-  def destroy_job(job_id, now)
-    run(:destroy_job, [], [job_id, now.utc.to_f])
-  end
-
-  def tickle_strand(job_id, strand, now)
-    run(:tickle_strand, [], [job_id, strand, now.utc.to_f])
-  end
-
-  def fail_job(job_id)
-    run(:fail_job, [], [job_id])
-  end
-
-  def set_running(job_id)
-    run(:set_running, [], [job_id])
-  end
-
-  def bulk_update(action, ids, flavor, query, now)
-    ids = (ids || []).join(",")
-    run(:bulk_update, [], [action, ids, flavor, query, now.utc.to_f])
-  end
-
-end
-end

data/lib/delayed/backend/redis/get_and_lock_next_available.lua

@@ -1,17 +0,0 @@
-local queue, min_priority, max_priority, worker_name, now = unpack(ARGV)
-local job_id = find_available(queue, 1, 0, min_priority, max_priority, now)[1]
-
-if job_exists(job_id) then
-  -- update the job with locked_by and locked_at
-  redis.call('HMSET', Keys.job(job_id), 'locked_by', worker_name, 'locked_at', now)
-
-  -- add the job to the running_jobs set
-  redis.call('ZADD', Keys.running_jobs(), now, job_id)
-  -- remove the job from the pending jobs queue
-  redis.call('ZREM', Keys.queue(queue), job_id)
-
-  -- return the list of job attributes
-  return redis.call('HGETALL', Keys.job(job_id))
-else
-  return {}
-end

data/lib/delayed/backend/redis/includes/jobs_common.lua

@@ -1,203 +0,0 @@
--- Keys holds the various functions to map to redis keys
--- These are duplicated from job.rb
-local Keys = {}
-
-Keys.job = function(id)
-  return "job/" .. id
-end
-
-Keys.running_jobs = function()
-  return "running_jobs"
-end
-
-Keys.failed_jobs = function()
-  return "failed_jobs"
-end
-
-Keys.queue = function(queue)
-  return "queue/" .. (queue or '')
-end
-
-Keys.future_queue = function(queue)
-  return Keys.queue(queue) .. "/future"
-end
-
-Keys.strand = function(strand_name)
-  if strand_name and string.len(strand_name) > 0 then
-    return "strand/" .. strand_name
-  else
-    return nil
-  end
-end
-
-Keys.tag_counts = function(flavor)
-  return "tag_counts/" .. flavor
-end
-
-Keys.tag = function(tag)
-  return "tag/" .. tag
-end
-
-Keys.waiting_strand_job_priority = function()
-  return 2000000
-end
-
--- remove the given job from the various queues
-local remove_from_queues = function(job_id, queue, strand)
-  local tag = unpack(redis.call('HMGET', Keys.job(job_id), 'tag'))
-
-  redis.call("SREM", Keys.tag(tag), job_id)
-
-  local current_delta = -redis.call('ZREM', Keys.queue(queue), job_id)
-  redis.call('ZREM', Keys.running_jobs(), job_id)
-  local future_delta = -redis.call('ZREM', Keys.future_queue(queue), job_id)
-
-  if current_delta ~= 0 then
-    redis.call('ZINCRBY', Keys.tag_counts('current'), current_delta, tag)
-  end
-
-  local total_delta = current_delta + future_delta
-
-  if total_delta ~= 0 then
-    redis.call('ZINCRBY', Keys.tag_counts('all'), total_delta, tag)
-  end
-
-  local strand_key = Keys.strand(strand)
-  if strand_key then
-    redis.call('LREM', strand_key, 1, job_id)
-  end
-end
-
--- returns the id for the first job on the strand, or nil if none
-local strand_next_job_id = function(strand)
-  local strand_key = Keys.strand(strand)
-  if not strand_key then return nil end
-  return redis.call('LRANGE', strand_key, 0, 0)[1]
-end
-
--- returns next_in_strand -- whether this added job is at the front of the strand
-local add_to_strand = function(job_id, strand)
-  local strand_key = Keys.strand(strand)
-  if not strand_key then return end
-  redis.call('RPUSH', strand_key, job_id) -- add to strand list
-  local next_id = strand_next_job_id(strand)
-  return next_id == job_id
-end
-
--- add this given job to the correct queues based on its state and the current time
--- also updates the tag counts and tag job lists
-local add_to_queues = function(job_id, queue, now)
-  local run_at, priority, tag, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'run_at', 'priority', 'tag', 'strand'))
-
-  redis.call("SADD", Keys.tag(tag), job_id)
-
-  if strand then
-    local next_job_id = strand_next_job_id(strand)
-    if next_job_id and next_job_id ~= job_id then
-      priority = Keys.waiting_strand_job_priority()
-    end
-  end
-
-  local current_delta = 0
-  local future_delta = 0
-
-  if run_at > now then
-    future_delta = future_delta + redis.call('ZADD', Keys.future_queue(queue), run_at, job_id)
-    current_delta = current_delta - redis.call('ZREM', Keys.queue(queue), job_id)
-  else
-    -- floor the run_at so we don't have a float in our float
-    local sort_key = priority .. '.' .. math.floor(run_at)
-    current_delta = current_delta + redis.call('ZADD', Keys.queue(queue), sort_key, job_id)
-    future_delta = future_delta - redis.call('ZREM', Keys.future_queue(queue), job_id)
-  end
-
-  if current_delta ~= 0 then
-    redis.call('ZINCRBY', Keys.tag_counts('current'), current_delta, tag)
-  end
-
-  local total_delta = current_delta + future_delta
-
-  if total_delta ~= 0 then
-    redis.call('ZINCRBY', Keys.tag_counts('all'), total_delta, tag)
-  end
-end
-
-local job_exists = function(job_id)
-  return job_id and redis.call('HGET', Keys.job(job_id), 'id')
-end
-
--- find jobs available for running
--- checks the future queue too, and moves and now-ready jobs
--- into the current queue
-local find_available = function(queue, limit, offset, min_priority, max_priority, now)
-  local ready_future_jobs = redis.call('ZRANGEBYSCORE', Keys.future_queue(queue), 0, now, 'limit', 0, limit)
-  for i, job_id in ipairs(ready_future_jobs) do
-    add_to_queues(job_id, queue, now)
-  end
-
-  if not min_priority or min_priority == '' then
-    min_priority = '0'
-  end
-
-  if not max_priority or max_priority == '' then
-    max_priority = "(" .. Keys.waiting_strand_job_priority()
-  else
-    max_priority = "(" .. (max_priority + 1)
-  end
-  local job_ids = redis.call('ZRANGEBYSCORE', Keys.queue(queue), min_priority, max_priority, 'limit', offset, limit)
-  for idx = table.getn(job_ids), 1, -1 do
-    local job_id = job_ids[idx]
-    if not job_exists(job_id) then
-      table.remove(job_ids, idx)
-      redis.call('ZREM', Keys.queue(queue), job_id)
-    end
-  end
-  return job_ids
-end
-
--- "tickle" the strand, removing the given job_id and setting the job at the
--- front of the strand as eligible to run, if it's not already
-local tickle_strand = function(job_id, strand, now)
-  local strand_key = Keys.strand(strand)
-
-  -- this LREM could be (relatively) slow if the strand is very large and this
-  -- job isn't near the front. however, in normal usage, we only delete from the
-  -- front. also the linked list is in memory, so even with thousands of jobs on
-  -- the strand it'll be quite fast.
-  --
-  -- alternatively we could make strands sorted sets, which would avoid a
-  -- linear search to delete this job. jobs need to be sorted on insertion
-  -- order though, and we're using GUIDs for keys here rather than an
-  -- incrementing integer, so we'd have to use an artificial counter as the
-  -- sort key (through `incrby strand_name` probably).
-  redis.call('LREM', strand_key, 1, job_id)
-  -- normally this loop will only run once, but we loop so that if there's any
-  -- job ids on the strand that don't actually exist anymore, we'll throw them
-  -- out and keep searching until we find a legit job or the strand is empty
-  while true do
-    local next_id = redis.call('LRANGE', strand_key, 0, 0)[1]
-    if next_id == nil then
-      break
-    elseif job_exists(next_id) then
-      -- technically jobs on the same strand can be in different queues,
-      -- though that functionality isn't currently used
-      local queue = redis.call('HGET', Keys.job(next_id), 'queue')
-      add_to_queues(next_id, queue, now)
-      break
-    else
-      redis.call('LPOP', strand_key)
-    end
-  end
-end
-
-local destroy_job = function(job_id, now)
-  local queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'strand'))
-  remove_from_queues(job_id, queue, strand)
-
-  if Keys.strand(strand) then
-    tickle_strand(job_id, strand, now)
-  end
-
-  redis.call('ZREM', Keys.failed_jobs(), job_id)
-  redis.call('DEL', Keys.job(job_id))
-end

data/lib/delayed/backend/redis/job.rb

@@ -1,535 +0,0 @@
-# frozen_string_literal: true
-
-# This can't currently be made compatible with redis cluster, because the Lua functions
-# access keys that aren't in their keys argument list (since they pop jobs off
-# a queue and then update the job with that id).
-
-# still TODO:
-#   * a consequence of our ignore-redis-failures code is that if redis is unavailable, creating delayed jobs silently fails, which is probably not what we want
-#   * need a way to migrate between jobs backends
-#   * we need some auditors:
-#     * fail jobs in running_jobs if they've timed out
-#     * have pools audit their workers and immediately fail jobs locked by dead workers (make sure this handles the restart case where two pools are running)
-#     * have a master auditor that fails jobs if a whole pool dies
-#     * audit strands ocasionally, look for any stuck strands where the strand queue isn't empty but there's no strand job running or queued
-module Delayed::Backend::Redis
-require 'delayed/backend/redis/functions'
-
-class Job
-  extend ActiveModel::Callbacks
-  define_model_callbacks :create, :save
-  include ActiveModel::Dirty
-  include Delayed::Backend::Base
-  # This redis instance needs to be set by the application during jobs configuration
-  cattr_accessor :redis
-
-  # An overview of where and when things are stored in redis:
-  #
-  # Jobs are given a UUID for an id, rather than an incrementing integer id.
-  # The job attributes are then stored in a redis hash at job/<id>. Attribute
-  # values are generally stored as their json representation, except for
-  # timestamps, which as stored as floating point utc-time-since-unix-epoch
-  # values, so that we can compare timestamps in Lua without a date parser.
-  #
-  # Jobs that are schedule to run immediately (in the present/past) are
-  # inserted into the queue named queue/<queue_name>. The queue is a sorted
-  # set, with the value being the job id and the weight being a floating point
-  # value, <priority>.<run_at>. This formatting is key to efficient
-  # querying of the next job to run.
-  #
-  # Jobs that are scheduled to run in the future are not inserted into the
-  # queue, but rather a future queue named queue/<queue_name>/future. This
-  # queue is also a sorted set, with the value being the job id, but the weight
-  # is just the <run_at> value.
-  #
-  # If the job is on a strand, the flow is different. First, it's inserted into
-  # a list named strand/<strand>. When strand jobs are inserted into the
-  # current jobs queue, we check if they're next to run in the strand. If not,
-  # we give them a special priority that is greater than MAX_PRIORITY, so that
-  # they won't run.  When a strand job is finished, failed or deleted,
-  # "tickle_strand" is called, which removes that job from the list and if that
-  # job was at the front of the list, changes the priority on the next job so
-  # that it's eligible to run.
-  #
-  # For singletons, the flow is the same as for other strand jobs, except that
-  # the job is thrown out if there are already any non-running jobs in the
-  # strand list.
-  #
-  # If a job fails, it's removed from the normal queues and inserted into the
-  # failed_jobs sorted set, with job id as the value and failure time as the
-  # key. The hash of job attributes is also renamed from job/<id> to
-  # failed_job/<id> -- use Delayed::Job::Failed to query those jobs, same as
-  # with AR jobs.
-  #
-  # We also insert into some other data structures for admin functionality.
-  # tag_counts/current and tag_counts/all are sorted sets storing the count of
-  # jobs for each tag. tag/<tag> is a set of existing job ids that have that tag.
-  #
-  # Most all of this happens in Lua functions, for atomicity. See the other
-  # files in this directory -- functions.rb is a wrapper to call the lua
-  # functions, and the individual functions are defined in .lua files in this
-  # directory.
-
-  # these key mappings are duplicated in the redis lua code, in include.lua
-  module Keys
-    RUNNING_JOBS = "running_jobs"
-    FAILED_JOBS = "failed_jobs"
-    JOB = proc { |id| "job/#{id}" }
-    FAILED_JOB = proc { |id| "failed_job/#{id}" }
-    QUEUE = proc { |name| "queue/#{name}" }
-    FUTURE_QUEUE = proc { |name| "#{QUEUE[name]}/future" }
-    STRAND = proc { |strand| strand ? "strand/#{strand}" : nil }
-    TAG_COUNTS = proc { |flavor| "tag_counts/#{flavor}" }
-    TAG = proc { |tag| "tag/#{tag}" }
-  end
-
-  WAITING_STRAND_JOB_PRIORITY = 2000000
-  if WAITING_STRAND_JOB_PRIORITY <= Delayed::MAX_PRIORITY
-    # if you change this, note that the value is duplicated in include.lua
-    raise("Delayed::MAX_PRIORITY must be less than #{WAITING_STRAND_JOB_PRIORITY}")
-  end
-
-  COLUMNS = []
-
-  # We store time attributes in redis as floats so we don't have to do
-  # timestamp parsing in lua.
-  TIMESTAMP_COLUMNS = []
-  INTEGER_COLUMNS = []
-
-  def self.column(name, type)
-    COLUMNS << name
-
-    if type == :timestamp
-      TIMESTAMP_COLUMNS << name
-    elsif type == :integer
-      INTEGER_COLUMNS << name
-    end
-
-    attr_reader(name)
-    define_attribute_methods([name])
-    # Custom attr_writer that updates the dirty status.
-    class_eval(<<-EOS, __FILE__, __LINE__ + 1)
-      def #{name}=(new_value)
-        #{name}_will_change! unless new_value == self.#{name}
-        @#{name} = new_value
-      end
-    EOS
-  end
-
-  column(:id, :string)
-  column(:priority, :integer)
-  column(:attempts, :integer)
-  column(:handler, :string)
-  column(:last_error, :string)
-  column(:queue, :string)
-  column(:run_at, :timestamp)
-  column(:locked_at, :timestamp)
-  column(:failed_at, :timestamp)
-  column(:locked_by, :string)
-  column(:created_at, :timestamp)
-  column(:updated_at, :timestamp)
-  column(:tag, :string)
-  column(:max_attempts, :integer)
-  column(:strand, :string)
-  column(:source, :string)
-  column(:expires_at, :timestamp)
-
-  def initialize(attrs = {})
-    attrs.each { |k, v| self.send("#{k}=", v) }
-    self.priority ||= 0
-    self.attempts ||= 0
-    @new_record = true
-  end
-
-  def self.instantiate(attrs)
-    result = new(attrs)
-    result.instance_variable_set(:@new_record, false)
-    result
-  end
-
-  def self.create(attrs = {})
-    result = new(attrs)
-    result.save
-    result
-  end
-
-  def self.create!(attrs = {})
-    result = new(attrs)
-    result.save!
-    result
-  end
-
-  def [](key)
-    send(key)
-  end
-
-  def []=(key, value)
-    send("#{key}=", value)
-  end
-
-  def self.find(ids)
-    if Array === ids
-      find_some(ids, {})
-    else
-      find_one(ids, {})
-    end
-  end
-
-  def new_record?
-    !!@new_record
-  end
-
-  def destroyed?
-    !!@destroyed
-  end
-
-  def ==(other)
-    other.is_a?(self.class) && id == other.id
-  end
-
-  def hash
-    id.hash
-  end
-
-  def self.reconnect!
-    # redis cluster responds to reconnect directly,
-    # but individual redis needs it to be called on client
-    redis.respond_to?(:reconnect) ?
-      redis.reconnect :
-      redis.client.reconnect
-  end
-
-  def self.functions
-    @@functions ||= Delayed::Backend::Redis::Functions.new(redis)
-  end
-
-  def self.find_one(id, options)
-    job = self.get_with_ids([id]).first
-    job || raise(ActiveRecord::RecordNotFound, "Couldn't find Job with ID=#{id}")
-  end
-
-  def self.find_some(ids, options)
-    self.get_with_ids(ids).compact
-  end
-
-  def self.get_with_ids(ids)
-    ids.map { |id| self.instantiate_from_attrs(redis.hgetall(key_for_job_id(id))) }
-  end
-
-  def self.key_for_job_id(job_id)
-    Keys::JOB[job_id]
-  end
-
-  def self.get_and_lock_next_available(worker_name,
-      queue = Delayed::Settings.queue,
-      min_priority = Delayed::MIN_PRIORITY,
-      max_priority = Delayed::MAX_PRIORITY,
-      prefetch: nil,
-      prefetch_owner: nil,
-      forced_latency: nil)
-
-    check_queue(queue)
-    check_priorities(min_priority, max_priority)
-    if worker_name.is_a?(Array)
-      multiple_workers = true
-      worker_name = worker_name.first
-    end
-
-    # as an optimization this lua function returns the hash of job attributes,
-    # rather than just a job id, saving a round trip
-    now = db_time_now
-    now -= forced_latency if forced_latency
-    job_attrs = functions.get_and_lock_next_available(worker_name, queue, min_priority, max_priority, now)
-    job = instantiate_from_attrs(job_attrs) # will return nil if the attrs are blank
-    if multiple_workers
-      if job.nil?
-        job = {}
-      else
-        job = { worker_name => job }
-      end
-    end
-    job
-  end
-
-  def self.find_available(limit,
-      queue = Delayed::Settings.queue,
-      min_priority = Delayed::MIN_PRIORITY,
-      max_priority = Delayed::MAX_PRIORITY)
-
-    check_queue(queue)
-    check_priorities(min_priority, max_priority)
-
-    self.find(functions.find_available(queue, limit, 0, min_priority, max_priority, db_time_now))
-  end
-
-  # get a list of jobs of the given flavor in the given queue
-  # flavor is :current, :future, :failed, :strand or :tag
-  # depending on the flavor, query has a different meaning:
-  # for :current and :future, it's the queue name (defaults to Delayed::Settings.queue)
-  # for :strand it's the strand name
-  # for :tag it's the tag name
-  # for :failed it's ignored
-  def self.list_jobs(flavor,
-      limit,
-      offset = 0,
-      query = nil)
-    case flavor.to_s
-      when 'current'
-        query ||= Delayed::Settings.queue
-        check_queue(query)
-        self.find(functions.find_available(query, limit, offset, 0, "+inf", db_time_now))
-      when 'future'
-        query ||= Delayed::Settings.queue
-        check_queue(query)
-        self.find(redis.zrangebyscore(Keys::FUTURE_QUEUE[query], 0, "+inf", :limit => [offset, limit]))
-      when 'failed'
-        Failed.find(redis.zrevrangebyscore(Keys::FAILED_JOBS, "+inf", 0, :limit => [offset, limit]))
-      when 'strand'
-        self.find(redis.lrange(Keys::STRAND[query], offset, offset + limit - 1))
-      when 'tag'
-        # This is optimized for writing, since list_jobs(:tag) will only ever happen in the admin UI
-        ids = redis.smembers(Keys::TAG[query])
-        self.find(ids[offset, limit])
-      else
-        raise ArgumentError, "invalid flavor: #{flavor.inspect}"
-    end
-  end
-
-  # get the total job count for the given flavor
-  # flavor is :current, :future or :failed
-  # for the :failed flavor, queue is currently ignored
-  def self.jobs_count(flavor,
-      queue = Delayed::Settings.queue)
-    case flavor.to_s
-      when 'current'
-        check_queue(queue)
-        redis.zcard(Keys::QUEUE[queue])
-      when 'future'
-        check_queue(queue)
-        redis.zcard(Keys::FUTURE_QUEUE[queue])
-      when 'failed'
-        redis.zcard(Keys::FAILED_JOBS)
-      else
-        raise ArgumentError, "invalid flavor: #{flavor.inspect}"
-    end
-  end
-
-  def self.strand_size(strand)
-    redis.llen(Keys::STRAND[strand])
-  end
-
-  def self.running_jobs()
-    self.find(redis.zrangebyscore(Keys::RUNNING_JOBS, 0, "+inf"))
-  end
-
-  def self.clear_locks!(worker_name)
-    self.running_jobs.each do |job|
-      # TODO: mark the job as failed one attempt
-      job.unlock! if job.locked_by == worker_name
-    end
-    nil
-  end
-
-  # returns a list of hashes { :tag => tag_name, :count => current_count }
-  # in descending count order
-  # flavor is :current or :all
-  def self.tag_counts(flavor,
-      limit,
-      offset = 0)
-    raise(ArgumentError, "invalid flavor: #{flavor.inspect}") unless %w(current all).include?(flavor.to_s)
-    key = Keys::TAG_COUNTS[flavor]
-    redis.zrevrangebyscore(key, '+inf', 1, :limit => [offset, limit], :withscores => true).map { |tag, count| { :tag => tag, :count => count } }
-  end
-
-  # perform a bulk update of a set of jobs
-  # action is :hold, :unhold, or :destroy
-  # to specify the jobs to act on, either pass opts[:ids] = [list of job ids]
-  # or opts[:flavor] = <some flavor> to perform on all jobs of that flavor
-  #
-  # see the list_jobs action for the list of available flavors and the meaning
-  # of opts[:query] for each
-  def self.bulk_update(action, opts)
-    if %w(current future).include?(opts[:flavor].to_s)
-      opts[:query] ||= Delayed::Settings.queue
-    end
-    functions.bulk_update(action, opts[:ids], opts[:flavor], opts[:query], db_time_now)
-  end
-
-  def self.create_singleton(options)
-    self.create!(options.merge(:singleton => true))
-  end
-
-  def self.unlock(jobs)
-    jobs.each(&:unlock!)
-    jobs.length
-  end
-
-  # not saved, just used as a marker when creating
-  attr_accessor :singleton, :on_conflict
-
-  def transfer_lock!(from:, to:)
-    lock_in_redis!(to)
-  end
-
-  def lock_in_redis!(worker_name)
-    self.locked_at = self.class.db_time_now
-    self.locked_by = worker_name
-    save
-  end
-
-  def unlock!
-    unlock
-    save!
-  end
-
-  def save(*a)
-    return false if destroyed?
-    result = run_callbacks(:save) do
-      if new_record?
-        run_callbacks(:create) { create }
-      else
-        update
-      end
-    end
-    changes_applied
-    result
-  end
-
-  if Rails.version < "4.1"
-    def changes_applied
-      @previously_changed = changes
-      @changed_attributes.clear
-    end
-  end
-
-  def save!(*a)
-    save(*a) || raise(RecordNotSaved)
-  end
-
-  def destroy
-    self.class.functions.destroy_job(id, self.class.db_time_now)
-    @destroyed = true
-    freeze
-  end
-
-  # take this job off the strand, and queue up the next strand job if this job
-  # was at the front
-  def tickle_strand
-    if strand.present?
-      self.class.functions.tickle_strand(id, strand, self.class.db_time_now)
-    end
-  end
-
-  def create_and_lock!(worker_name)
-    raise "job already exists" unless new_record?
-    lock_in_redis!(worker_name)
-  end
-
-  def fail!
-    self.failed_at = self.class.db_time_now
-    save!
-    redis.rename Keys::JOB[id], Keys::FAILED_JOB[id]
-    tickle_strand
-    self
-  end
-
-  protected
-
-  def update_queues
-    if failed_at
-      self.class.functions.fail_job(id)
-    elsif locked_at
-      self.class.functions.set_running(id)
-    elsif singleton
-      job_id = self.class.functions.create_singleton(id, queue, strand, self.class.db_time_now)
-      # if create_singleton returns a different job id, that means this job got
-      # deleted because there was already that other job on the strand. so
-      # replace this job with the other for returning.
-      if job_id != self.id
-        singleton = self.class.find(job_id)
-
-        self.on_conflict ||= :use_earliest
-        singleton.run_at =
-          case self.on_conflict
-          when :use_earliest
-            [singleton.run_at, run_at].min
-          when :overwrite
-            run_at
-          when :loose
-            singleton.run_at
-          end
-        singleton.handler = self.handler if self.on_conflict == :overwrite
-        singleton.save! if singleton.changed?
-        COLUMNS.each { |c| send("#{c}=", singleton.send(c)) }
-      end
-    else
-      self.class.functions.enqueue(id, queue, strand, self.class.db_time_now)
-    end
-  end
-
-  def create
-    self.id ||= SecureRandom.hex(16)
-    self.created_at = self.updated_at = Time.now.utc
-    save_job_to_redis
-    update_queues
-
-    @new_record = false
-    self.id
-  end
-
-  def update
-    self.updated_at = Time.now.utc
-    save_job_to_redis
-    update_queues
-    true
-  end
-
-  def queue_score
-    "#{priority}.#{run_at.to_i}".to_f
-  end
-
-  def save_job_to_redis
-    to_delete = []
-    attrs = {}
-    COLUMNS.each do |k|
-      v = send(k)
-      if v.nil?
-        to_delete << k if !new_record? && changed.include?(k.to_s)
-      elsif v.is_a?(ActiveSupport::TimeWithZone) || v.is_a?(Time)
-        attrs[k] = v.utc.to_f
-      else
-        attrs[k] = v.as_json
-      end
-    end
-    key = Keys::JOB[id]
-    redis.mapped_hmset(key, attrs)
-    redis.hdel(key, to_delete) unless to_delete.empty?
-  end
-
-  def self.instantiate_from_attrs(redis_attrs)
-    if redis_attrs['id'].present?
-      attrs = redis_attrs.with_indifferent_access
-      TIMESTAMP_COLUMNS.each { |k| attrs[k] = Time.zone.at(attrs[k].to_f) if attrs[k] }
-      INTEGER_COLUMNS.each { |k| attrs[k] = attrs[k].to_i if attrs[k] }
-      instantiate(attrs)
-    else
-      nil
-    end
-  end
-
-  def global_id
-    id
-  end
-
-  class Failed < Job
-    include Delayed::Backend::Base
-    def self.key_for_job_id(job_id)
-      Keys::FAILED_JOB[job_id]
-    end
-
-    def original_job_id
-      id
-    end
-  end
-end
-end