qless 0.9.1

data/lib/qless/qless-core/tag.lua

@@ -0,0 +1,100 @@
+-- tag(0, ('add' | 'remove'), jid, now, tag, [tag, ...])
+-- tag(0, 'get', tag, [offset, [count]])
+-- tag(0, 'top', [offset, [count]])
+-- ------------------------------------------------------------------------------------------------------------------
+-- Accepts a jid, 'add' or 'remove', and then a list of tags
+-- to either add or remove from the job. Alternatively, 'get',
+-- a tag to get jobs associated with that tag, and offset and
+-- count
+--
+-- If 'add' or 'remove', the response is a list of the jobs
+-- current tags, or False if the job doesn't exist. If 'get',
+-- the response is of the form:
+--
+--	{
+--		total: ...,
+--		jobs: [
+--			jid,
+--			...
+--		]
+--	}
+--
+-- If 'top' is supplied, it returns the most commonly-used tags
+-- in a paginated fashion.
+
+if #KEYS ~= 0 then
+	error('Tag(): Got ' .. #KEYS .. ', expected 0')
+end
+
+local command = assert(ARGV[1], 'Tag(): Missing first arg "add", "remove" or "get"')
+
+if command == 'add' then
+	local jid  = assert(ARGV[2]          , 'Tag(): Arg "jid" missing')
+	local now  = assert(tonumber(ARGV[3]), 'Tag(): Arg "now" is not a number')
+	local tags = redis.call('hget', 'ql:j:' .. jid, 'tags')
+	-- If the job has been canceled / deleted, then return false
+	if tags then
+		-- Decode the json blob, convert to dictionary
+		tags = cjson.decode(tags)
+		local _tags = {}
+		for i,v in ipairs(tags) do _tags[v] = true end
+	
+		-- Otherwise, add the job to the sorted set with that tags
+		for i=4,#ARGV do
+			local tag = ARGV[i]
+			if _tags[tag] == nil then
+				table.insert(tags, tag)
+			end
+			redis.call('zadd', 'ql:t:' .. tag, now, jid)
+			redis.call('zincrby', 'ql:tags', 1, tag)
+		end
+	
+		tags = cjson.encode(tags)
+		redis.call('hset', 'ql:j:' .. jid, 'tags', tags)
+		return tags
+	else
+		return false
+	end
+elseif command == 'remove' then
+	local jid  = assert(ARGV[2]          , 'Tag(): Arg "jid" missing')
+	local now  = assert(tonumber(ARGV[3]), 'Tag(): Arg "now" is not a number')
+	local tags = redis.call('hget', 'ql:j:' .. jid, 'tags')
+	-- If the job has been canceled / deleted, then return false
+	if tags then
+		-- Decode the json blob, convert to dictionary
+		tags = cjson.decode(tags)
+		local _tags = {}
+		for i,v in ipairs(tags) do _tags[v] = true end
+	
+		-- Otherwise, add the job to the sorted set with that tags
+		for i=4,#ARGV do
+			local tag = ARGV[i]
+			_tags[tag] = nil
+			redis.call('zrem', 'ql:t:' .. tag, jid)
+			redis.call('zincrby', 'ql:tags', -1, tag)
+		end
+	
+		local results = {}
+		for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
+	
+		tags = cjson.encode(results)
+		redis.call('hset', 'ql:j:' .. jid, 'tags', tags)
+		return tags
+	else
+		return false
+	end
+elseif command == 'get' then
+	local tag    = assert(ARGV[2]                , 'Tag(): Arg "tag" missing')
+	local offset = assert(tonumber(ARGV[3] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(ARGV[3]))
+	local count  = assert(tonumber(ARGV[4] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(ARGV[4]))
+	return cjson.encode({
+		total = redis.call('zcard', 'ql:t:' .. tag),
+		jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, count)
+	})
+elseif command == 'top' then
+	local offset = assert(tonumber(ARGV[2] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(ARGV[2]))
+	local count  = assert(tonumber(ARGV[3] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(ARGV[3]))
+	return cjson.encode(redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count))
+else
+	error('Tag(): First argument must be "add", "remove" or "get"')
+end

data/lib/qless/qless-core/track.lua

@@ -0,0 +1,79 @@
+-- Track(0)
+-- Track(0, ('track' | 'untrack'), jid, now)
+-- ------------------------------------------
+-- If no arguments are provided, it returns details of all currently-tracked jobs.
+-- If the first argument is 'track', then it will start tracking the job associated
+-- with that id, and 'untrack' stops tracking it. In this context, tracking is
+-- nothing more than saving the job to a list of jobs that are considered special.
+-- __Returns__ JSON:
+-- 
+-- 	{
+-- 		'jobs': [
+-- 			{
+-- 				'jid': ...,
+-- 				# All the other details you'd get from 'get'
+-- 			}, {
+-- 				...
+-- 			}
+-- 		], 'expired': [
+-- 			# These are all the jids that are completed and whose data expired
+-- 			'deadbeef',
+-- 			...,
+-- 			...,
+-- 		]
+-- 	}
+--
+
+if #KEYS ~= 0 then
+	error('Track(): No keys expected. Got ' .. #KEYS)
+end
+
+if ARGV[1] ~= nil then
+	local jid = assert(ARGV[2]          , 'Track(): Arg "jid" missing')
+	local now = assert(tonumber(ARGV[3]), 'Track(): Arg "now" missing or not a number: ' .. (ARGV[3] or 'nil'))
+	if string.lower(ARGV[1]) == 'track' then
+		redis.call('publish', 'track', jid)
+		return redis.call('zadd', 'ql:tracked', now, jid)
+	elseif string.lower(ARGV[1]) == 'untrack' then
+		redis.call('publish', 'untrack', jid)
+		return redis.call('zrem', 'ql:tracked', jid)
+	else
+		error('Track(): Unknown action "' .. ARGV[1] .. '"')
+	end
+else
+	local response = {
+		jobs = {},
+		expired = {}
+	}
+	local jids = redis.call('zrange', 'ql:tracked', 0, -1)
+	for index, jid in ipairs(jids) do
+		local job = redis.call(
+		    'hmget', 'ql:j:' .. jid, 'jid', 'klass', 'state', 'queue', 'worker', 'priority',
+			'expires', 'retries', 'remaining', 'data', 'tags', 'history', 'failure')
+		
+		if job[1] then
+			table.insert(response.jobs, {
+			    jid          = job[1],
+				klass        = job[2],
+			    state        = job[3],
+			    queue        = job[4],
+				worker       = job[5] or '',
+				tracked      = true,
+				priority     = tonumber(job[6]),
+				expires      = tonumber(job[7]) or 0,
+				retries      = tonumber(job[8]),
+				remaining    = tonumber(job[9]),
+				data         = cjson.decode(job[10]),
+				tags         = cjson.decode(job[11]),
+			    history      = cjson.decode(job[12]),
+				failure      = cjson.decode(job[13] or '{}'),
+				dependents   = redis.call('smembers', 'ql:j:' .. jid .. '-dependents'),
+				dependencies = redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')
+				
+			})
+		else
+			table.insert(response.expired, jid)
+		end
+	end
+	return cjson.encode(response)
+end

data/lib/qless/qless-core/workers.lua

@@ -0,0 +1,69 @@
+-- Workers(0, now, [worker])
+-- -------------------------
+-- Provide data about all the workers, or if a specific worker is provided, then
+-- which jobs that worker is responsible for. If no worker is provided, expect a
+-- response of the form:
+-- 
+-- 	[
+-- 		# This is sorted by the recency of activity from that worker
+-- 		{
+-- 			'name'   : 'hostname1-pid1',
+-- 			'jobs'   : 20,
+-- 			'stalled': 0
+-- 		}, {
+-- 			...
+-- 		}
+-- 	]
+-- 
+-- If a worker id is provided, then expect a response of the form:
+-- 
+-- 	{
+-- 		'jobs': [
+-- 			jid1,
+-- 			jid2,
+-- 			...
+-- 		], 'stalled': [
+-- 			jid1,
+-- 			...
+-- 		]
+-- 	}
+--
+if #KEYS > 0 then
+	error('Workers(): No key arguments expected')
+end
+
+local now = assert(tonumber(ARGV[1]), 'Workers(): Arg "now" missing or not a number: ' .. (ARGV[1] or 'nil'))
+
+-- Clean up all the workers' job lists if they're too old. This is determined
+-- by the `max-worker-age` configuration, defaulting to the last day. Seems
+-- like a 'reasonable' default
+local interval = tonumber(
+	redis.call('hget', 'ql:config', 'max-worker-age')) or 86400
+
+local workers  = redis.call('zrangebyscore', 'ql:workers', 0, now - interval)
+for index, worker in ipairs(workers) do
+	redis.call('del', 'ql:w:' .. worker .. ':jobs')
+end
+
+-- And now remove them from the list of known workers
+redis.call('zremrangebyscore', 'ql:workers', 0, now - interval)
+
+if #ARGV == 1 then
+	local response = {}
+	local workers = redis.call('zrevrange', 'ql:workers', 0, -1)
+	for index, worker in ipairs(workers) do
+		table.insert(response, {
+			name    = worker,
+			jobs    = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000),
+			stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now)
+		})
+	end
+	return cjson.encode(response)
+else
+	local worker = assert(ARGV[2], 'Workers(): Arg "worker" missing.')
+	local response = {
+		jobs    = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now),
+		stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0)
+	}
+	return cjson.encode(response)
+end

data/lib/qless/queue.rb

@@ -0,0 +1,141 @@
+require "qless/lua"
+require "qless/job"
+require "redis"
+require "json"
+
+module Qless
+  class QueueJobs
+    def initialize(name, client)
+      @name   = name
+      @client = client
+    end
+    
+    def running(start=0, count=25)
+      @client._jobs.call([], ['running', Time.now.to_f, @name, start, count])
+    end
+    
+    def stalled(start=0, count=25)
+      @client._jobs.call([], ['stalled', Time.now.to_f, @name, start, count])
+    end
+    
+    def scheduled(start=0, count=25)
+      @client._jobs.call([], ['scheduled', Time.now.to_f, @name, start, count])
+    end
+    
+    def depends(start=0, count=25)
+      @client._jobs.call([], ['depends', Time.now.to_f, @name, start, count])
+    end
+    
+    def recurring(start=0, count=25)
+      @client._jobs.call([], ['recurring', Time.now.to_f, @name, start, count])
+    end
+  end
+  
+  class Queue
+    attr_reader   :name
+    attr_accessor :worker_name
+    
+    def initialize(name, client)
+      @client = client
+      @name   = name
+      self.worker_name = Qless.worker_name
+    end
+    
+    def jobs
+      @jobs ||= QueueJobs.new(@name, @client)
+    end
+    
+    def counts
+      JSON.parse(@client._queues.call([], [Time.now.to_i, @name]))
+    end
+    
+    def heartbeat
+      @client.config["#{@name}-heartbeat"]
+    end
+    
+    def heartbeat=(value)
+      @client.config["#{@name}-heartbeat"] = value
+    end
+    
+    # Put the described job in this queue
+    # Options include:
+    # => priority (int)
+    # => tags (array of strings)
+    # => delay (int)
+    def put(klass, data, opts={})
+      opts = job_options(klass, data, opts)
+
+      @client._put.call([@name], [
+        (opts[:jid] or Qless.generate_jid),
+        klass.name,
+        JSON.generate(data),
+        Time.now.to_f,
+        opts.fetch(:delay, 0),
+        'priority', opts.fetch(:priority, 0),
+        'tags', JSON.generate(opts.fetch(:tags, [])),
+        'retries', opts.fetch(:retries, 5),
+        'depends', JSON.generate(opts.fetch(:depends, []))
+      ])
+    end
+    
+    # Make a recurring job in this queue
+    # Options include:
+    # => priority (int)
+    # => tags (array of strings)
+    # => retries (int)
+    # => offset (int)
+    def recur(klass, data, interval, opts={})
+      opts = job_options(klass, data, opts)
+
+      @client._recur.call([], [
+        'on',
+        @name,
+        (opts[:jid] or Qless.generate_jid),
+        klass.to_s,
+        JSON.generate(data),
+        Time.now.to_f,
+        'interval', interval, opts.fetch(:offset, 0),
+        'priority', opts.fetch(:priority, 0),
+        'tags', JSON.generate(opts.fetch(:tags, [])),
+        'retries', opts.fetch(:retries, 5)
+      ])
+    end
+    
+    # Pop a work item off the queue
+    def pop(count=nil)
+      results = @client._pop.call([@name], [worker_name, (count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
+      count.nil? ? results[0] : results
+    end
+    
+    # Peek at a work item
+    def peek(count=nil)
+      results = @client._peek.call([@name], [(count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
+      count.nil? ? results[0] : results
+    end
+    
+    def stats(date=nil)
+      JSON.parse(@client._stats.call([], [@name, (date || Time.now.to_f)]))
+    end
+    
+    # How many items in the queue?
+    def length
+      (@client.redis.multi do
+        @client.redis.zcard("ql:q:#{@name}-locks")
+        @client.redis.zcard("ql:q:#{@name}-work")
+        @client.redis.zcard("ql:q:#{@name}-scheduled")
+      end).inject(0, :+)
+    end
+
+    def to_s
+      "#<Qless::Queue #{@name}>"
+    end
+    alias inspect to_s
+
+  private
+
+    def job_options(klass, data, opts)
+      return opts unless klass.respond_to?(:default_job_options)
+      klass.default_job_options(data).merge(opts)
+    end
+  end
+end

data/lib/qless/server.rb

@@ -0,0 +1,411 @@
+require 'sinatra/base'
+require 'qless'
+
+# Much of this is shamelessly poached from the resque web client
+
+module Qless
+  class Server < Sinatra::Base
+    # Path-y-ness
+    dir = File.dirname(File.expand_path(__FILE__))
+    set :views        , "#{dir}/server/views"
+    set :public_folder, "#{dir}/server/static"
+    
+    # For debugging purposes at least, I want this
+    set :reload_templates, true
+    
+    # I'm not sure what this option is -- I'll look it up later
+    # set :static, true
+    
+    def self.client
+      @client ||= Qless::Client.new
+    end
+
+    def self.client=(client)
+      @client = client
+    end
+    
+    helpers do
+      include Rack::Utils
+
+      def url_path(*path_parts)
+        [ path_prefix, path_parts ].join("/").squeeze('/')
+      end
+      alias_method :u, :url_path
+
+      def path_prefix
+        request.env['SCRIPT_NAME']
+      end
+      
+      def tabs
+        return [
+          {:name => 'Queues'  , :path => '/queues'  },
+          {:name => 'Workers' , :path => '/workers' },
+          {:name => 'Track'   , :path => '/track'   },
+          {:name => 'Failed'  , :path => '/failed'  },
+          {:name => 'Config'  , :path => '/config'  },
+          {:name => 'About'   , :path => '/about'   }
+        ]
+      end
+      
+      def application_name
+        return Server.client.config['application']
+      end
+      
+      def queues
+        return Server.client.queues.counts
+      end
+      
+      def tracked
+        return Server.client.jobs.tracked
+      end
+      
+      def workers
+        return Server.client.workers.counts
+      end
+      
+      def failed
+        return Server.client.jobs.failed
+      end
+      
+      # Return the supplied object back as JSON
+      def json(obj)
+        content_type :json
+        obj.to_json
+      end
+      
+      # Make the id acceptable as an id / att in HTML
+      def sanitize_attr(attr)
+        return attr.gsub(/[^a-zA-Z\:\_]/, '-')
+      end
+      
+      # What are the top tags? Since it might go on, say, every
+      # page, then we should probably be caching it
+      def top_tags
+        @top_tags ||= {
+          :top     => Server.client.tags,
+          :fetched => Time.now
+        }
+        if (Time.now - @top_tags[:fetched]) > 60 then
+          @top_tags = {
+            :top     => Server.client.tags,
+            :fetched => Time.now
+          }
+        end
+        @top_tags[:top]
+      end
+      
+      def strftime(t)
+        # From http://stackoverflow.com/questions/195740/how-do-you-do-relative-time-in-rails
+        diff_seconds = Time.now - t
+        case diff_seconds
+          when 0 .. 59
+            "#{diff_seconds.to_i} seconds ago"
+          when 60 ... 3600
+            "#{(diff_seconds/60).to_i} minutes ago"
+          when 3600 ... 3600*24
+            "#{(diff_seconds/3600).to_i} hours ago"
+          when (3600*24) ... (3600*24*30) 
+            "#{(diff_seconds/(3600*24)).to_i} days ago"
+          else
+            t.strftime('%b %e, %Y %H:%M:%S %Z (%z)')
+        end
+      end
+    end
+    
+    get '/?' do
+      erb :overview, :layout => true, :locals => { :title => "Overview" }
+    end
+    
+    # Returns a JSON blob with the job counts for various queues
+    get '/queues.json' do
+      json(Server.client.queues.counts)
+    end
+    
+    get '/queues/?' do
+      erb :queues, :layout => true, :locals => {
+        :title   => 'Queues'
+      }
+    end
+    
+    # Return the job counts for a specific queue
+    get '/queues/:name.json' do
+      json(Server.client.queues[params[:name]].counts)
+    end
+    
+    get '/queues/:name/?:tab?' do
+      queue = Server.client.queues[params[:name]]
+      tab    = params.fetch('tab', 'stats')
+      jobs   = []
+      case tab
+      when 'running'
+        jobs = queue.jobs.running
+      when 'scheduled'
+        jobs = queue.jobs.scheduled
+      when 'stalled'
+        jobs = queue.jobs.stalled
+      when 'depends'
+        jobs = queue.jobs.depends
+      when 'recurring'
+        jobs = queue.jobs.recurring
+      end
+      jobs = jobs.map { |jid| Server.client.jobs[jid] }
+      if tab == 'waiting'
+        jobs = queue.peek(20)
+      end
+      erb :queue, :layout => true, :locals => {
+        :title   => "Queue #{params[:name]}",
+        :tab     => tab,
+        :jobs    => jobs,
+        :queue   => Server.client.queues[params[:name]].counts,
+        :stats   => queue.stats
+      }
+    end
+    
+    get '/failed/?' do
+      # qless-core doesn't provide functionality this way, so we'll
+      # do it ourselves. I'm not sure if this is how the core library
+      # should behave or not.
+      erb :failed, :layout => true, :locals => {
+        :title  => 'Failed',
+        :failed => Server.client.jobs.failed.keys.map { |t| Server.client.jobs.failed(t).tap { |f| f['type'] = t } }
+      }
+    end
+    
+    get '/failed/:type/?' do
+      erb :failed_type, :layout => true, :locals => {
+        :title  => 'Failed | ' + params[:type],
+        :type   => params[:type],
+        :failed => Server.client.jobs.failed(params[:type])
+      }
+    end
+    
+    get '/track/?' do
+      erb :track, :layout => true, :locals => {
+        :title   => 'Track'
+      }
+    end
+    
+    get '/jobs/:jid' do
+      erb :job, :layout => true, :locals => {
+        :title => "Job | #{params[:jid]}",
+        :jid   => params[:jid],
+        :job   => Server.client.jobs[params[:jid]]
+      }
+    end
+    
+    get '/workers/?' do
+      erb :workers, :layout => true, :locals => {
+        :title   => 'Workers'
+      }
+    end
+
+    get '/workers/:worker' do
+      erb :worker, :layout => true, :locals => {
+        :title  => 'Worker | ' + params[:worker],
+        :worker => Server.client.workers[params[:worker]].tap { |w|
+          w['jobs']    = w['jobs'].map { |j| Server.client.jobs[j] }
+          w['stalled'] = w['stalled'].map { |j| Server.client.jobs[j] }
+          w['name']    = params[:worker]
+        }
+      }
+    end
+    
+    get '/tag/?' do
+      jobs = Server.client.jobs.tagged(params[:tag])
+      erb :tag, :layout => true, :locals => {
+        :title => "Tag | #{params[:tag]}",
+        :tag   => params[:tag],
+        :jobs  => jobs['jobs'].map { |jid| Server.client.jobs[jid] },
+        :total => jobs['total']
+      }
+    end
+    
+    get '/config/?' do
+      erb :config, :layout => true, :locals => {
+        :title   => 'Config',
+        :options => Server.client.config.all
+      }
+    end
+    
+    get '/about/?' do
+      erb :about, :layout => true, :locals => {
+        :title   => 'About'
+      }
+    end
+    
+    
+    
+    
+    
+    
+    
+    # These are the bits where we accept AJAX requests
+    post "/track/?" do
+      # Expects a JSON-encoded hash with a job id, and optionally some tags
+      data = JSON.parse(request.body.read)
+      job = Server.client.jobs[data["id"]]
+      if not job.nil?
+        data.fetch("tags", false) ? job.track(*data["tags"]) : job.track()
+        if request.xhr?
+          json({ :tracked => [job.jid] })
+        else
+          redirect to('/track')
+        end
+      else
+        if request.xhr?
+          json({ :tracked => [] })
+        else
+          redirect to(request.referrer)
+        end
+      end
+    end
+    
+    post "/untrack/?" do
+      # Expects a JSON-encoded array of job ids to stop tracking
+      jobs = JSON.parse(request.body.read).map { |jid| Server.client.jobs[jid] }.select { |j| not j.nil? }
+      # Go ahead and cancel all the jobs!
+      jobs.each do |job|
+        job.untrack()
+      end
+      return json({ :untracked => jobs.map { |job| job.jid } })
+    end
+    
+    post "/priority/?" do
+      # Expects a JSON-encoded dictionary of jid => priority
+      response = Hash.new
+      r = JSON.parse(request.body.read)
+      r.each_pair do |jid, priority|
+        begin
+          Server.client.jobs[jid].priority = priority
+          response[jid] = priority
+        rescue
+          response[jid] = 'failed'
+        end
+      end
+      return json(response)
+    end
+    
+    post "/tag/?" do
+      # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
+      response = Hash.new
+      JSON.parse(request.body.read).each_pair do |jid, tags|
+        begin
+          Server.client.jobs[jid].tag(*tags)
+          response[jid] = tags
+        rescue
+          response[jid] = 'failed'
+        end
+      end
+      return json(response)
+    end
+    
+    post "/untag/?" do
+      # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
+      response = Hash.new
+      JSON.parse(request.body.read).each_pair do |jid, tags|
+        begin
+          Server.client.jobs[jid].untag(*tags)
+          response[jid] = tags
+        rescue
+          response[jid] = 'failed'
+        end
+      end
+      return json(response)
+    end
+    
+    post "/move/?" do
+      # Expects a JSON-encoded hash of id: jid, and queue: queue_name
+      data = JSON.parse(request.body.read)
+      if data["id"].nil? or data["queue"].nil?
+        halt 400, "Need id and queue arguments"
+      else
+        job = Server.client.jobs[data["id"]]
+        if job.nil?
+          halt 404, "Could not find job"
+        else
+          job.move(data["queue"])
+          return json({ :id => data["id"], :queue => data["queue"]})
+        end
+      end
+    end
+    
+    post "/undepend/?" do
+      # Expects a JSON-encoded hash of id: jid, and queue: queue_name
+      data = JSON.parse(request.body.read)
+      if data["id"].nil?
+        halt 400, "Need id"
+      else
+        job = Server.client.jobs[data["id"]]
+        if job.nil?
+          halt 404, "Could not find job"
+        else
+          job.undepend(data['dependency'])
+          return json({:id => data["id"]})
+        end
+      end
+    end
+    
+    post "/retry/?" do
+      # Expects a JSON-encoded hash of id: jid, and queue: queue_name
+      data = JSON.parse(request.body.read)
+      if data["id"].nil?
+        halt 400, "Need id"
+      else
+        job = Server.client.jobs[data["id"]]
+        if job.nil?
+          halt 404, "Could not find job"
+        else
+          queue = job.history[-1]["q"]
+          job.move(queue)
+          return json({ :id => data["id"], :queue => queue})
+        end
+      end
+    end
+    
+    # Retry all the failures of a particular type
+    post "/retryall/?" do
+      # Expects a JSON-encoded hash of type: failure-type
+      data = JSON.parse(request.body.read)
+      if data["type"].nil?
+        halt 400, "Neet type"
+      else
+        return json(Server.client.jobs.failed(data["type"], 0, 500)['jobs'].map do |job|
+          queue = job.history[-1]["q"]
+          job.move(queue)
+          { :id => job.jid, :queue => queue}
+        end)
+      end
+    end
+    
+    post "/cancel/?" do
+      # Expects a JSON-encoded array of job ids to cancel
+      jobs = JSON.parse(request.body.read).map { |jid| Server.client.jobs[jid] }.select { |j| not j.nil? }
+      # Go ahead and cancel all the jobs!
+      jobs.each do |job|
+        job.cancel()
+      end
+      
+      if request.xhr?
+        return json({ :canceled => jobs.map { |job| job.jid } })
+      else
+        redirect to(request.referrer)
+      end
+    end
+    
+    post "/cancelall/?" do
+      # Expects a JSON-encoded hash of type: failure-type
+      data = JSON.parse(request.body.read)
+      if data["type"].nil?
+        halt 400, "Neet type"
+      else
+        return json(Server.client.jobs.failed(data["type"])['jobs'].map do |job|
+          job.cancel()
+          { :id => job.jid }
+        end)
+      end
+    end
+    
+    # start the server if ruby file executed directly
+    run! if app_file == $0
+  end
+end