canvas-jobs 0.9.0

data/lib/delayed/backend/redis/bulk_update.lua

@@ -0,0 +1,40 @@
+local action, id_string, flavor, query, now = unpack(ARGV)
+
+local ids = {}
+
+if string.len(flavor) > 0 then
+  if flavor == 'current' then
+    ids = redis.call('ZRANGE', Keys.queue(query), 0, -1)
+  elseif flavor == 'future' then
+    ids = redis.call('ZRANGE', Keys.future_queue(query), 0, -1)
+  elseif flavor == 'strand' then
+    ids = redis.call('LRANGE', Keys.strand(query), 0, -1)
+  elseif flavor == 'tag' then
+    ids = redis.call('SMEMBERS', Keys.tag(query))
+  end
+else
+  -- can't pass an array to redis/lua, so we split the string here
+  for id in string.gmatch(id_string, "([%w-]+)") do
+    if job_exists(id) then
+      table.insert(ids, id)
+    end
+  end
+end
+
+for idx, job_id in ipairs(ids) do
+  if action == 'hold' then
+    local queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'strand'))
+    remove_from_queues(job_id, queue, strand)
+    redis.call('HMSET', Keys.job(job_id), 'locked_at', now, 'locked_by', 'on hold', 'attempts', 50)
+  elseif action == 'unhold' then
+    local queue, locked_by = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'locked_by'))
+    add_to_queues(job_id, queue, now)
+    redis.call('HDEL', Keys.job(job_id), 'locked_at', 'locked_by')
+    redis.call('HMSET', Keys.job(job_id), 'attempts', 0)
+  elseif action == 'destroy' then
+    destroy_job(job_id, now)
+  end
+end
+
+-- returns the # of jobs matching the query, not necessarily the # whose state was changed
+return table.getn(ids)

data/lib/delayed/backend/redis/destroy_job.lua

@@ -0,0 +1,2 @@
+local job_id, now = unpack(ARGV)
+destroy_job(job_id, now)

data/lib/delayed/backend/redis/enqueue.lua

@@ -0,0 +1,29 @@
+local job_id, queue, strand, now, for_singleton = unpack(ARGV)
+local strand_key = Keys.strand(strand)
+
+-- if this is a singleton job, only queue it up if another doesn't exist on the strand
+-- otherwise, delete it and return the other job id
+if for_singleton then
+  local job_ids = redis.call('LRANGE', strand_key, 0, 1)
+  local job_to_check = 1
+  if job_exists(job_ids[1]) and redis.call('HGET', Keys.job(job_ids[1]), 'locked_at') then
+    job_to_check = 2
+  end
+
+  local job_to_check_id = job_ids[job_to_check]
+  if job_exists(job_to_check_id) then
+    -- delete the new job, we found a match
+    redis.call('DEL', Keys.job(job_id))
+    return job_to_check_id
+  end
+end
+
+-- if this job is in a strand, add it to the strand queue first
+-- if it's not at the front of the strand, we won't enqueue it below
+if strand_key then
+  add_to_strand(job_id, strand)
+end
+
+add_to_queues(job_id, queue, now)
+
+return job_id

data/lib/delayed/backend/redis/fail_job.lua

@@ -0,0 +1,5 @@
+local job_id = unpack(ARGV)
+local failed_at, queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'failed_at', 'queue', 'strand'))
+
+remove_from_queues(job_id, queue, strand)
+redis.call('ZADD', Keys.failed_jobs(), failed_at, job_id)

data/lib/delayed/backend/redis/find_available.lua

@@ -0,0 +1,3 @@
+local queue, limit, offset, min_priority, max_priority, now = unpack(ARGV)
+
+return find_available(queue, limit, offset, min_priority, max_priority, now)

data/lib/delayed/backend/redis/functions.rb

@@ -0,0 +1,57 @@
+require 'redis/scripting'
+
+# This module handles loading the Lua functions into Redis and running them
+module Delayed::Backend::Redis
+class Functions < ::Redis::Scripting::Module
+  def initialize(redis)
+    super(redis, File.dirname(__FILE__))
+  end
+
+  def run_script(script, keys, argv)
+    result = nil
+    ms = Benchmark.ms { result = super }
+    line = 'Redis Jobs Timing: %s (%.1fms)' % [script.name, ms]
+    ActiveRecord::Base.logger.debug(line)
+    result
+  end
+
+  def find_available(queue, limit, offset, min_priority, max_priority, now)
+    run(:find_available, [], [queue, limit, offset, min_priority, max_priority, now.utc.to_f])
+  end
+
+  def get_and_lock_next_available(worker_name, queue, min_priority, max_priority, now)
+    attrs = run(:get_and_lock_next_available, [], [queue, min_priority, max_priority, worker_name, now.utc.to_f])
+    Hash[*attrs]
+  end
+
+  def enqueue(job_id, queue, strand, now)
+    run(:enqueue, [], [job_id, queue, strand, now.utc.to_f])
+  end
+
+  def create_singleton(job_id, queue, strand, now)
+    run(:enqueue, [], [job_id, queue, strand, now.utc.to_f, true])
+  end
+
+  def destroy_job(job_id, now)
+    run(:destroy_job, [], [job_id, now.utc.to_f])
+  end
+
+  def tickle_strand(job_id, strand, now)
+    run(:tickle_strand, [], [job_id, strand, now.utc.to_f])
+  end
+
+  def fail_job(job_id)
+    run(:fail_job, [], [job_id])
+  end
+
+  def set_running(job_id)
+    run(:set_running, [], [job_id])
+  end
+
+  def bulk_update(action, ids, flavor, query, now)
+    ids = (ids || []).join(",")
+    run(:bulk_update, [], [action, ids, flavor, query, now.utc.to_f])
+  end
+
+end
+end

data/lib/delayed/backend/redis/get_and_lock_next_available.lua

@@ -0,0 +1,17 @@
+local queue, min_priority, max_priority, worker_name, now = unpack(ARGV)
+local job_id = find_available(queue, 1, 0, min_priority, max_priority, now)[1]
+
+if job_exists(job_id) then
+  -- update the job with locked_by and locked_at
+  redis.call('HMSET', Keys.job(job_id), 'locked_by', worker_name, 'locked_at', now)
+
+  -- add the job to the running_jobs set
+  redis.call('ZADD', Keys.running_jobs(), now, job_id)
+  -- remove the job from the pending jobs queue
+  redis.call('ZREM', Keys.queue(queue), job_id)
+
+  -- return the list of job attributes
+  return redis.call('HGETALL', Keys.job(job_id))
+else
+  return {}
+end

data/lib/delayed/backend/redis/includes/jobs_common.lua

@@ -0,0 +1,203 @@
+-- Keys holds the various functions to map to redis keys
+-- These are duplicated from job.rb
+local Keys = {}
+
+Keys.job = function(id)
+  return "job/" .. id
+end
+
+Keys.running_jobs = function()
+  return "running_jobs"
+end
+
+Keys.failed_jobs = function()
+  return "failed_jobs"
+end
+
+Keys.queue = function(queue)
+  return "queue/" .. (queue or '')
+end
+
+Keys.future_queue = function(queue)
+  return Keys.queue(queue) .. "/future"
+end
+
+Keys.strand = function(strand_name)
+  if strand_name and string.len(strand_name) > 0 then
+    return "strand/" .. strand_name
+  else
+    return nil
+  end
+end
+
+Keys.tag_counts = function(flavor)
+  return "tag_counts/" .. flavor
+end
+
+Keys.tag = function(tag)
+  return "tag/" .. tag
+end
+
+Keys.waiting_strand_job_priority = function()
+  return 2000000
+end
+
+-- remove the given job from the various queues
+local remove_from_queues = function(job_id, queue, strand)
+  local tag = unpack(redis.call('HMGET', Keys.job(job_id), 'tag'))
+
+  redis.call("SREM", Keys.tag(tag), job_id)
+
+  local current_delta = -redis.call('ZREM', Keys.queue(queue), job_id)
+  redis.call('ZREM', Keys.running_jobs(), job_id)
+  local future_delta = -redis.call('ZREM', Keys.future_queue(queue), job_id)
+
+  if current_delta ~= 0 then
+    redis.call('ZINCRBY', Keys.tag_counts('current'), current_delta, tag)
+  end
+
+  local total_delta = current_delta + future_delta
+
+  if total_delta ~= 0 then
+    redis.call('ZINCRBY', Keys.tag_counts('all'), total_delta, tag)
+  end
+
+  local strand_key = Keys.strand(strand)
+  if strand_key then
+    redis.call('LREM', strand_key, 1, job_id)
+  end
+end
+
+-- returns the id for the first job on the strand, or nil if none
+local strand_next_job_id = function(strand)
+  local strand_key = Keys.strand(strand)
+  if not strand_key then return nil end
+  return redis.call('LRANGE', strand_key, 0, 0)[1]
+end
+
+-- returns next_in_strand -- whether this added job is at the front of the strand
+local add_to_strand = function(job_id, strand)
+  local strand_key = Keys.strand(strand)
+  if not strand_key then return end
+  redis.call('RPUSH', strand_key, job_id) -- add to strand list
+  local next_id = strand_next_job_id(strand)
+  return next_id == job_id
+end
+
+-- add this given job to the correct queues based on its state and the current time
+-- also updates the tag counts and tag job lists
+local add_to_queues = function(job_id, queue, now)
+  local run_at, priority, tag, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'run_at', 'priority', 'tag', 'strand'))
+
+  redis.call("SADD", Keys.tag(tag), job_id)
+
+  if strand then
+    local next_job_id = strand_next_job_id(strand)
+    if next_job_id and next_job_id ~= job_id then
+      priority = Keys.waiting_strand_job_priority()
+    end
+  end
+
+  local current_delta = 0
+  local future_delta = 0
+
+  if run_at > now then
+    future_delta = future_delta + redis.call('ZADD', Keys.future_queue(queue), run_at, job_id)
+    current_delta = current_delta - redis.call('ZREM', Keys.queue(queue), job_id)
+  else
+    -- floor the run_at so we don't have a float in our float
+    local sort_key = priority .. '.' .. math.floor(run_at)
+    current_delta = current_delta + redis.call('ZADD', Keys.queue(queue), sort_key, job_id)
+    future_delta = future_delta - redis.call('ZREM', Keys.future_queue(queue), job_id)
+  end
+
+  if current_delta ~= 0 then
+    redis.call('ZINCRBY', Keys.tag_counts('current'), current_delta, tag)
+  end
+
+  local total_delta = current_delta + future_delta
+
+  if total_delta ~= 0 then
+    redis.call('ZINCRBY', Keys.tag_counts('all'), total_delta, tag)
+  end
+end
+
+local job_exists = function(job_id)
+  return job_id and redis.call('HGET', Keys.job(job_id), 'id')
+end
+
+-- find jobs available for running
+-- checks the future queue too, and moves and now-ready jobs
+-- into the current queue
+local find_available = function(queue, limit, offset, min_priority, max_priority, now)
+  local ready_future_jobs = redis.call('ZRANGEBYSCORE', Keys.future_queue(queue), 0, now, 'limit', 0, limit)
+  for i, job_id in ipairs(ready_future_jobs) do
+    add_to_queues(job_id, queue, now)
+  end
+
+  if not min_priority or min_priority == '' then
+    min_priority = '0'
+  end
+
+  if not max_priority or max_priority == '' then
+    max_priority = "+inf"
+  else
+    max_priority = "(" .. (max_priority + 1)
+  end
+  local job_ids = redis.call('ZRANGEBYSCORE', Keys.queue(queue), min_priority, max_priority, 'limit', offset, limit)
+  for idx = table.getn(job_ids), 1, -1 do
+    local job_id = job_ids[idx]
+    if not job_exists(job_id) then
+      table.remove(job_ids, idx)
+      redis.call('ZREM', Keys.queue(queue), job_id)
+    end
+  end
+  return job_ids
+end
+
+-- "tickle" the strand, removing the given job_id and setting the job at the
+-- front of the strand as eligible to run, if it's not already
+local tickle_strand = function(job_id, strand, now)
+  local strand_key = Keys.strand(strand)
+
+  -- this LREM could be (relatively) slow if the strand is very large and this
+  -- job isn't near the front. however, in normal usage, we only delete from the
+  -- front. also the linked list is in memory, so even with thousands of jobs on
+  -- the strand it'll be quite fast.
+  --
+  -- alternatively we could make strands sorted sets, which would avoid a
+  -- linear search to delete this job. jobs need to be sorted on insertion
+  -- order though, and we're using GUIDs for keys here rather than an
+  -- incrementing integer, so we'd have to use an artificial counter as the
+  -- sort key (through `incrby strand_name` probably).
+  redis.call('LREM', strand_key, 1, job_id)
+  -- normally this loop will only run once, but we loop so that if there's any
+  -- job ids on the strand that don't actually exist anymore, we'll throw them
+  -- out and keep searching until we find a legit job or the strand is empty
+  while true do
+    local next_id = redis.call('LRANGE', strand_key, 0, 0)[1]
+    if next_id == nil then
+      break
+    elseif job_exists(next_id) then
+      -- technically jobs on the same strand can be in different queues,
+      -- though that functionality isn't currently used
+      local queue = redis.call('HGET', Keys.job(next_id), 'queue')
+      add_to_queues(next_id, queue, now)
+      break
+    else
+      redis.call('LPOP', strand_key)
+    end
+  end
+end
+
+local destroy_job = function(job_id, now)
+  local queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'queue', 'strand'))
+  remove_from_queues(job_id, queue, strand)
+
+  if Keys.strand(strand) then
+    tickle_strand(job_id, strand, now)
+  end
+
+  redis.call('ZREM', Keys.failed_jobs(), job_id)
+  redis.call('DEL', Keys.job(job_id))
+end

data/lib/delayed/backend/redis/job.rb

@@ -0,0 +1,481 @@
+# This can't currently be made compatible with redis cluster, because the Lua functions
+# access keys that aren't in their keys argument list (since they pop jobs off
+# a queue and then update the job with that id).
+
+# still TODO:
+#   * a consequence of our ignore-redis-failures code is that if redis is unavailable, creating delayed jobs silently fails, which is probably not what we want
+#   * need a way to migrate between jobs backends
+#   * we need some auditors:
+#     * fail jobs in running_jobs if they've timed out
+#     * have pools audit their workers and immediately fail jobs locked by dead workers (make sure this handles the restart case where two pools are running)
+#     * have a master auditor that fails jobs if a whole pool dies
+#     * audit strands ocasionally, look for any stuck strands where the strand queue isn't empty but there's no strand job running or queued
+module Delayed::Backend::Redis
+require 'delayed/backend/redis/functions'
+
+class Job
+  extend ActiveModel::Callbacks
+  define_model_callbacks :create, :save
+  include ActiveModel::Dirty
+  include Delayed::Backend::Base
+  # This redis instance needs to be set by the application during jobs configuration
+  cattr_accessor :redis
+
+  # An overview of where and when things are stored in redis:
+  #
+  # Jobs are given a UUID for an id, rather than an incrementing integer id.
+  # The job attributes are then stored in a redis hash at job/<id>. Attribute
+  # values are generally stored as their json representation, except for
+  # timestamps, which as stored as floating point utc-time-since-unix-epoch
+  # values, so that we can compare timestamps in Lua without a date parser.
+  #
+  # Jobs that are schedule to run immediately (in the present/past) are
+  # inserted into the queue named queue/<queue_name>. The queue is a sorted
+  # set, with the value being the job id and the weight being a floating point
+  # value, <priority>.<run_at>. This formatting is key to efficient
+  # querying of the next job to run.
+  #
+  # Jobs that are scheduled to run in the future are not inserted into the
+  # queue, but rather a future queue named queue/<queue_name>/future. This
+  # queue is also a sorted set, with the value being the job id, but the weight
+  # is just the <run_at> value.
+  #
+  # If the job is on a strand, the flow is different. First, it's inserted into
+  # a list named strand/<strand>. When strand jobs are inserted into the
+  # current jobs queue, we check if they're next to run in the strand. If not,
+  # we give them a special priority that is greater than MAX_PRIORITY, so that
+  # they won't run.  When a strand job is finished, failed or deleted,
+  # "tickle_strand" is called, which removes that job from the list and if that
+  # job was at the front of the list, changes the priority on the next job so
+  # that it's eligible to run.
+  #
+  # For singletons, the flow is the same as for other strand jobs, except that
+  # the job is thrown out if there are already any non-running jobs in the
+  # strand list.
+  #
+  # If a job fails, it's removed from the normal queues and inserted into the
+  # failed_jobs sorted set, with job id as the value and failure time as the
+  # key. The hash of job attributes is also renamed from job/<id> to
+  # failed_job/<id> -- use Delayed::Job::Failed to query those jobs, same as
+  # with AR jobs.
+  #
+  # We also insert into some other data structures for admin functionality.
+  # tag_counts/current and tag_counts/all are sorted sets storing the count of
+  # jobs for each tag. tag/<tag> is a set of existing job ids that have that tag.
+  #
+  # Most all of this happens in Lua functions, for atomicity. See the other
+  # files in this directory -- functions.rb is a wrapper to call the lua
+  # functions, and the individual functions are defined in .lua files in this
+  # directory.
+
+  # these key mappings are duplicated in the redis lua code, in include.lua
+  module Keys
+    RUNNING_JOBS = "running_jobs"
+    FAILED_JOBS = "failed_jobs"
+    JOB = proc { |id| "job/#{id}" }
+    FAILED_JOB = proc { |id| "failed_job/#{id}" }
+    QUEUE = proc { |name| "queue/#{name}" }
+    FUTURE_QUEUE = proc { |name| "#{QUEUE[name]}/future" }
+    STRAND = proc { |strand| strand ? "strand/#{strand}" : nil }
+    TAG_COUNTS = proc { |flavor| "tag_counts/#{flavor}" }
+    TAG = proc { |tag| "tag/#{tag}" }
+  end
+
+  WAITING_STRAND_JOB_PRIORITY = 2000000
+  if WAITING_STRAND_JOB_PRIORITY <= Delayed::MAX_PRIORITY
+    # if you change this, note that the value is duplicated in include.lua
+    raise("Delayed::MAX_PRIORITY must be less than #{WAITING_STRAND_JOB_PRIORITY}")
+  end
+
+  COLUMNS = [
+    :id,
+    :priority,
+    :attempts,
+    :handler,
+    :last_error,
+    :queue,
+    :run_at,
+    :locked_at,
+    :failed_at,
+    :locked_by,
+    :created_at,
+    :updated_at,
+    :tag,
+    :max_attempts,
+    :strand,
+    :source,
+  ]
+
+  # We store time attributes in redis as floats so we don't have to do
+  # timestamp parsing in lua.
+  TIMESTAMP_COLUMNS = [:run_at, :locked_at, :failed_at, :created_at, :updated_at]
+  INTEGER_COLUMNS = [:priority, :attempts, :max_attempts]
+
+  attr_reader(*COLUMNS)
+  define_attribute_methods(COLUMNS)
+  COLUMNS.each do |c|
+    # Custom attr_writer that updates the dirty status.
+    class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+      def #{c}=(new_value)
+        #{c}_will_change! unless new_value == self.#{c}
+        @#{c} = new_value
+      end
+    EOS
+  end
+
+  def initialize(attrs = {})
+    attrs.each { |k, v| self.send("#{k}=", v) }
+    self.priority ||= 0
+    self.attempts ||= 0
+    @new_record = true
+  end
+
+  def self.instantiate(attrs)
+    result = new(attrs)
+    result.instance_variable_set(:@new_record, false)
+    result
+  end
+
+  def self.create(attrs = {})
+    result = new(attrs)
+    result.save
+    result
+  end
+
+  def self.create!(attrs = {})
+    result = new(attrs)
+    result.save!
+    result
+  end
+
+  def [](key)
+    send(key)
+  end
+
+  def []=(key, value)
+    send("#{key}=", value)
+  end
+
+  def self.find(ids)
+    if Array === ids
+      find_some(ids, {})
+    else
+      find_one(ids, {})
+    end
+  end
+
+  def new_record?
+    !!@new_record
+  end
+
+  def destroyed?
+    !!@destroyed
+  end
+
+  def ==(other)
+    other.is_a?(self.class) && id == other.id
+  end
+
+  def hash
+    id.hash
+  end
+
+  def self.reconnect!
+    self.redis.reconnect
+  end
+
+  def self.functions
+    @@functions ||= Delayed::Backend::Redis::Functions.new(redis)
+  end
+
+  def self.find_one(id, options)
+    job = self.get_with_ids([id]).first
+    job || raise(ActiveRecord::RecordNotFound, "Couldn't find Job with ID=#{id}")
+  end
+
+  def self.find_some(ids, options)
+    self.get_with_ids(ids).compact
+  end
+
+  def self.get_with_ids(ids)
+    ids.map { |id| self.instantiate_from_attrs(redis.hgetall(key_for_job_id(id))) }
+  end
+
+  def self.key_for_job_id(job_id)
+    Keys::JOB[job_id]
+  end
+
+  def self.get_and_lock_next_available(worker_name,
+      queue = Delayed::Settings.queue,
+      min_priority = Delayed::MIN_PRIORITY,
+      max_priority = Delayed::MAX_PRIORITY)
+
+    check_queue(queue)
+    check_priorities(min_priority, max_priority)
+
+    # as an optimization this lua function returns the hash of job attributes,
+    # rather than just a job id, saving a round trip
+    job_attrs = functions.get_and_lock_next_available(worker_name, queue, min_priority, max_priority, db_time_now)
+    instantiate_from_attrs(job_attrs) # will return nil if the attrs are blank
+  end
+
+  def self.find_available(limit,
+      queue = Delayed::Settings.queue,
+      min_priority = Delayed::MIN_PRIORITY,
+      max_priority = Delayed::MAX_PRIORITY)
+
+    check_queue(queue)
+    check_priorities(min_priority, max_priority)
+
+    self.find(functions.find_available(queue, limit, 0, min_priority, max_priority, db_time_now))
+  end
+
+  # get a list of jobs of the given flavor in the given queue
+  # flavor is :current, :future, :failed, :strand or :tag
+  # depending on the flavor, query has a different meaning:
+  # for :current and :future, it's the queue name (defaults to Delayed::Settings.queue)
+  # for :strand it's the strand name
+  # for :tag it's the tag name
+  # for :failed it's ignored
+  def self.list_jobs(flavor,
+      limit,
+      offset = 0,
+      query = nil)
+    case flavor.to_s
+      when 'current'
+        query ||= Delayed::Settings.queue
+        check_queue(query)
+        self.find(functions.find_available(query, limit, offset, nil, nil, db_time_now))
+      when 'future'
+        query ||= Delayed::Settings.queue
+        check_queue(query)
+        self.find(redis.zrangebyscore(Keys::FUTURE_QUEUE[query], 0, "+inf", :limit => [offset, limit]))
+      when 'failed'
+        Failed.find(redis.zrevrangebyscore(Keys::FAILED_JOBS, "+inf", 0, :limit => [offset, limit]))
+      when 'strand'
+        self.find(redis.lrange(Keys::STRAND[query], offset, offset + limit - 1))
+      when 'tag'
+        # This is optimized for writing, since list_jobs(:tag) will only ever happen in the admin UI
+        ids = redis.smembers(Keys::TAG[query])
+        self.find(ids[offset, limit])
+      else
+        raise ArgumentError, "invalid flavor: #{flavor.inspect}"
+    end
+  end
+
+  # get the total job count for the given flavor
+  # flavor is :current, :future or :failed
+  # for the :failed flavor, queue is currently ignored
+  def self.jobs_count(flavor,
+      queue = Delayed::Settings.queue)
+    case flavor.to_s
+      when 'current'
+        check_queue(queue)
+        redis.zcard(Keys::QUEUE[queue])
+      when 'future'
+        check_queue(queue)
+        redis.zcard(Keys::FUTURE_QUEUE[queue])
+      when 'failed'
+        redis.zcard(Keys::FAILED_JOBS)
+      else
+        raise ArgumentError, "invalid flavor: #{flavor.inspect}"
+    end
+  end
+
+  def self.strand_size(strand)
+    redis.llen(Keys::STRAND[strand])
+  end
+
+  def self.running_jobs()
+    self.find(redis.zrangebyscore(Keys::RUNNING_JOBS, 0, "+inf"))
+  end
+
+  def self.clear_locks!(worker_name)
+    self.running_jobs.each do |job|
+      # TODO: mark the job as failed one attempt
+      job.unlock! if job.locked_by == worker_name
+    end
+  end
+
+  # returns a list of hashes { :tag => tag_name, :count => current_count }
+  # in descending count order
+  # flavor is :current or :all
+  def self.tag_counts(flavor,
+      limit,
+      offset = 0)
+    raise(ArgumentError, "invalid flavor: #{flavor.inspect}") unless %w(current all).include?(flavor.to_s)
+    key = Keys::TAG_COUNTS[flavor]
+    redis.zrevrangebyscore(key, '+inf', 1, :limit => [offset, limit], :withscores => true).map { |tag, count| { :tag => tag, :count => count } }
+  end
+
+  # perform a bulk update of a set of jobs
+  # action is :hold, :unhold, or :destroy
+  # to specify the jobs to act on, either pass opts[:ids] = [list of job ids]
+  # or opts[:flavor] = <some flavor> to perform on all jobs of that flavor
+  #
+  # see the list_jobs action for the list of available flavors and the meaning
+  # of opts[:query] for each
+  def self.bulk_update(action, opts)
+    if %w(current future).include?(opts[:flavor].to_s)
+      opts[:query] ||= Delayed::Settings.queue
+    end
+    functions.bulk_update(action, opts[:ids], opts[:flavor], opts[:query], db_time_now)
+  end
+
+  def self.create_singleton(options)
+    self.create!(options.merge(:singleton => true))
+  end
+
+  # not saved, just used as a marker when creating
+  attr_accessor :singleton
+
+  def lock_in_redis!(worker_name)
+    self.locked_at = self.class.db_time_now
+    self.locked_by = worker_name
+    save
+  end
+
+  def unlock!
+    self.locked_at = nil
+    self.locked_by = nil
+    save!
+  end
+
+  def save(*a)
+    return false if destroyed?
+    result = run_callbacks(:save) do
+      if new_record?
+        run_callbacks(:create) { create }
+      else
+        update
+      end
+    end
+    changes_applied
+    result
+  end
+
+  if Rails.version < "4.1"
+    def changes_applied
+      @previously_changed = changes
+      @changed_attributes.clear
+    end
+  end
+
+  def save!(*a)
+    save(*a) || raise(RecordNotSaved)
+  end
+
+  def destroy
+    self.class.functions.destroy_job(id, self.class.db_time_now)
+    @destroyed = true
+    freeze
+  end
+
+  # take this job off the strand, and queue up the next strand job if this job
+  # was at the front
+  def tickle_strand
+    if strand.present?
+      self.class.functions.tickle_strand(id, strand, self.class.db_time_now)
+    end
+  end
+
+  def create_and_lock!(worker_name)
+    raise "job already exists" unless new_record?
+    lock_in_redis!(worker_name)
+  end
+
+  def fail!
+    self.failed_at = self.class.db_time_now
+    save!
+    redis.rename Keys::JOB[id], Keys::FAILED_JOB[id]
+    tickle_strand
+    self
+  end
+
+  protected
+
+  def update_queues
+    if failed_at
+      self.class.functions.fail_job(id)
+    elsif locked_at
+      self.class.functions.set_running(id)
+    elsif singleton
+      job_id = self.class.functions.create_singleton(id, queue, strand, self.class.db_time_now)
+      # if create_singleton returns a different job id, that means this job got
+      # deleted because there was already that other job on the strand. so
+      # replace this job with the other for returning.
+      if job_id != self.id
+        singleton = self.class.find(job_id)
+        COLUMNS.each { |c| send("#{c}=", singleton.send(c)) }
+      end
+    else
+      self.class.functions.enqueue(id, queue, strand, self.class.db_time_now)
+    end
+  end
+
+  def create
+    self.id ||= SecureRandom.hex(16)
+    self.created_at = self.updated_at = Time.now.utc
+    save_job_to_redis
+    update_queues
+
+    @new_record = false
+    self.id
+  end
+
+  def update
+    self.updated_at = Time.now.utc
+    save_job_to_redis
+    update_queues
+    true
+  end
+
+  def queue_score
+    "#{priority}.#{run_at.to_i}".to_f
+  end
+
+  def save_job_to_redis
+    to_delete = []
+    attrs = {}
+    COLUMNS.each do |k|
+      v = send(k)
+      if v.nil?
+        to_delete << k if !new_record? && changed.include?(k.to_s)
+      elsif v.is_a?(ActiveSupport::TimeWithZone)
+        attrs[k] = v.utc.to_f
+      else
+        attrs[k] = v.as_json
+      end
+    end
+    key = Keys::JOB[id]
+    redis.mapped_hmset(key, attrs)
+    redis.hdel(key, to_delete) unless to_delete.empty?
+  end
+
+  def self.instantiate_from_attrs(redis_attrs)
+    if redis_attrs['id'].present?
+      attrs = redis_attrs.with_indifferent_access
+      TIMESTAMP_COLUMNS.each { |k| attrs[k] = Time.zone.at(attrs[k].to_f) if attrs[k] }
+      INTEGER_COLUMNS.each { |k| attrs[k] = attrs[k].to_i if attrs[k] }
+      instantiate(attrs)
+    else
+      nil
+    end
+  end
+
+  def global_id
+    id
+  end
+
+  class Failed < Job
+    include Delayed::Backend::Base
+    def self.key_for_job_id(job_id)
+      Keys::FAILED_JOB[job_id]
+    end
+
+    def original_job_id
+      id
+    end
+  end
+end
+end