qless 0.9.1

data/lib/qless/qless-core/queues.lua

@@ -0,0 +1,58 @@
+-- Queues(0, now, [queue])
+-- -----------------------
+-- 
+-- Return all the queues we know about, with how many jobs are scheduled, waiting,
+-- and running in that queue. If a queue name is provided, then only the appropriate
+-- response hash should be returned. The response is JSON:
+-- 
+-- 	[
+-- 		{
+-- 			'name': 'testing',
+-- 			'stalled': 2,
+-- 			'waiting': 5,
+-- 			'running': 5,
+-- 			'scheduled': 10,
+-- 			'depends': 5,
+-- 			'recurring': 0
+-- 		}, {
+-- 			...
+-- 		}
+-- 	]
+
+if #KEYS > 0 then
+	error('Queues(): Got '.. #KEYS .. ' expected 0 KEYS arguments')
+end
+
+local now   = assert(tonumber(ARGV[1]), 'Queues(): Arg "now" missing or not a number: ' .. (ARGV[1] or 'nil'))
+local queue = ARGV[2]
+
+local response = {}
+local queuenames = redis.call('zrange', 'ql:queues', 0, -1)
+
+if queue then
+	local stalled = redis.call('zcount', 'ql:q:' .. queue .. '-locks', 0, now)
+	response = {
+		name      = queue,
+		waiting   = redis.call('zcard', 'ql:q:' .. queue .. '-work'),
+		stalled   = stalled,
+		running   = redis.call('zcard', 'ql:q:' .. queue .. '-locks') - stalled,
+		scheduled = redis.call('zcard', 'ql:q:' .. queue .. '-scheduled'),
+		depends   = redis.call('zcard', 'ql:q:' .. queue .. '-depends'),
+		recurring = redis.call('zcard', 'ql:q:' .. queue .. '-recur')
+	}
+else
+	for index, qname in ipairs(queuenames) do
+		local stalled = redis.call('zcount', 'ql:q:' .. qname .. '-locks', 0, now)
+		table.insert(response, {
+			name      = qname,
+			waiting   = redis.call('zcard', 'ql:q:' .. qname .. '-work'),
+			stalled   = stalled,
+			running   = redis.call('zcard', 'ql:q:' .. qname .. '-locks') - stalled,
+			scheduled = redis.call('zcard', 'ql:q:' .. qname .. '-scheduled'),
+			depends   = redis.call('zcard', 'ql:q:' .. qname .. '-depends'),
+			recurring = redis.call('zcard', 'ql:q:' .. qname .. '-recur')
+		})
+	end
+end
+
+return cjson.encode(response)

data/lib/qless/qless-core/recur.lua

@@ -0,0 +1,181 @@
+-- Recur(0, 'on', queue, jid, klass, data, now, 'interval', second, offset, [priority p], [tags t], [retries r])
+-- Recur(0, 'off', jid)
+-- Recur(0, 'get', jid)
+-- Recur(0, 'update', jid, ['priority', priority], ['interval', interval], ['retries', retries], ['data', data], ['klass', klass], ['queue', queue])
+-- Recur(0, 'tag', jid, tag, [tag, [...]])
+-- Recur(0, 'untag', jid, tag, [tag, [...]])
+-- -------------------------------------------------------------------------------------------------------
+-- This script takes the name of a queue, and then the info
+-- info about the work item, and makes sure that jobs matching
+-- its criteria are regularly made available.
+
+if #KEYS ~= 0 then
+	error('Recur(): Got ' .. #KEYS .. ', expected 0 KEYS arguments')
+end
+
+local command  = assert(ARGV[1]               , 'Recur(): Missing first argument')
+
+if command == 'on' then
+	local queue    = assert(ARGV[2]               , 'Recur(): Arg "queue" missing')
+	local jid      = assert(ARGV[3]               , 'Recur(): Arg "jid" missing')
+	local klass    = assert(ARGV[4]               , 'Recur(): Arg "klass" missing')
+	local data     = assert(cjson.decode(ARGV[5]) , 'Recur(): Arg "data" missing or not JSON: '    .. tostring(ARGV[5]))
+	local now      = assert(tonumber(ARGV[6])     , 'Recur(): Arg "now" missing or not a number: ' .. tostring(ARGV[6]))
+	local spec     = assert(ARGV[7]               , 'Recur(): Arg "schedule type" missing')
+	if spec == 'interval' then
+		local interval = assert(tonumber(ARGV[8]) , 'Recur(): Arg "interval" must be a number: '   .. tostring(ARGV[8]))
+		local offset   = assert(tonumber(ARGV[9]) , 'Recur(): Arg "offset" must be a number: '     .. tostring(ARGV[9]))
+		if interval <= 0 then
+			error('Recur(): Arg "interval" must be greater than or equal to 0')
+		end
+		-- Read in all the optional parameters
+		local options = {}
+		for i = 10, #ARGV, 2 do options[ARGV[i]] = ARGV[i + 1] end
+		options.tags     = assert(cjson.decode(options.tags or {}), 'Recur(): Arg "tags" must be JSON-encoded array of string. Got: ' .. tostring(options.tags))
+		options.priority = assert(tonumber(options.priority or 0) , 'Recur(): Arg "priority" must be a number. Got: ' .. tostring(options.priority))
+		options.retries  = assert(tonumber(options.retries  or 0) , 'Recur(): Arg "retries" must be a number. Got: ' .. tostring(options.retries))
+		
+		-- Do some insertions
+		redis.call('hmset', 'ql:r:' .. jid,
+			'jid'     , jid,
+			'klass'   , klass,
+			'data'    , cjson.encode(data),
+			'priority', options.priority,
+			'tags'    , cjson.encode(options.tags or {}),
+			'state'   , 'recur',
+			'queue'   , queue,
+			'type'    , 'interval',
+			-- How many jobs we've spawned from this
+			'count'   , 0,
+			'interval', interval,
+			'retries' , options.retries)
+		-- Now, we should schedule the next run of the job
+		redis.call('zadd', 'ql:q:' .. queue .. '-recur', now + offset, jid)
+		
+		-- Lastly, we're going to make sure that this item is in the
+		-- set of known queues. We should keep this sorted by the 
+		-- order in which we saw each of these queues
+		if redis.call('zscore', 'ql:queues', queue) == false then
+			redis.call('zadd', 'ql:queues', now, queue)
+		end
+		
+		return jid
+	else
+		error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
+	end
+elseif command == 'off' then
+	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
+	-- First, find out what queue it was attached to
+	local queue = redis.call('hget', 'ql:r:' .. jid, 'queue')
+	if queue then
+		-- Now, delete it from the queue it was attached to, and delete the thing itself
+		redis.call('zrem', 'ql:q:' .. queue .. '-recur', jid)
+		redis.call('del', 'ql:r:' .. jid)
+		return true
+	else
+		return true
+	end
+elseif command == 'get' then
+	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
+	local job = redis.call(
+	    'hmget', 'ql:r:' .. jid, 'jid', 'klass', 'state', 'queue',
+		'priority', 'interval', 'retries', 'count', 'data', 'tags')
+	
+	if not job[1] then
+		return false
+	end
+	
+	return cjson.encode({
+	    jid          = job[1],
+		klass        = job[2],
+	    state        = job[3],
+	    queue        = job[4],
+		priority     = tonumber(job[5]),
+		interval     = tonumber(job[6]),
+		retries      = tonumber(job[7]),
+		count        = tonumber(job[8]),
+		data         = cjson.decode(job[9]),
+		tags         = cjson.decode(job[10])
+	})
+elseif command == 'update' then
+	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
+	local options = {}
+	
+	-- Make sure that the job exists
+	if redis.call('exists', 'ql:r:' .. jid) then
+		for i = 3, #ARGV, 2 do
+			local key   = ARGV[i]
+			local value = ARGV[i+1]
+			if key == 'priority' or key == 'interval' or key == 'retries' then
+				value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value))
+				-- If the command is 'interval', then we need to update the time
+				-- when it should next be scheduled
+				if key == 'interval' then
+					local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. jid, 'queue', 'interval'))
+					redis.call('zincrby', 'ql:q:' .. queue .. '-recur', value - tonumber(interval), jid)
+				end
+				redis.call('hset', 'ql:r:' .. jid, key, value)
+			elseif key == 'data' then
+				value = assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value))
+				redis.call('hset', 'ql:r:' .. jid, 'data', cjson.encode(value))
+			elseif key == 'klass' then
+				redis.call('hset', 'ql:r:' .. jid, 'klass', value)
+			elseif key == 'queue' then
+				local queue = redis.call('hget', 'ql:r:' .. jid, 'queue')
+				local score = redis.call('zscore', 'ql:q:' .. queue .. '-recur', jid)
+				redis.call('zrem', 'ql:q:' .. queue .. '-recur', jid)
+				redis.call('zadd', 'ql:q:' .. value .. '-recur', score, jid)
+				redis.call('hset', 'ql:r:' .. jid, 'queue', value)
+			else
+				error('Recur(): Unrecognized option "' .. key .. '"')
+			end
+		end
+		return true
+	else
+		return false
+	end
+elseif command == 'tag' then
+	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
+	local tags = redis.call('hget', 'ql:r:' .. jid, 'tags')
+	-- If the job has been canceled / deleted, then return false
+	if tags then
+		-- Decode the json blob, convert to dictionary
+		tags = cjson.decode(tags)
+		local _tags = {}
+		for i,v in ipairs(tags) do _tags[v] = true end
+		
+		-- Otherwise, add the job to the sorted set with that tags
+		for i=3,#ARGV do if _tags[ARGV[i]] == nil then table.insert(tags, ARGV[i]) end end
+		
+		tags = cjson.encode(tags)
+		redis.call('hset', 'ql:r:' .. jid, 'tags', tags)
+		return tags
+	else
+		return false
+	end
+elseif command == 'untag' then
+	local jid  = assert(ARGV[2], 'Recur(): Arg "jid" missing')
+	-- Get the existing tags
+	local tags = redis.call('hget', 'ql:r:' .. jid, 'tags')
+	-- If the job has been canceled / deleted, then return false
+	if tags then
+		-- Decode the json blob, convert to dictionary
+		tags = cjson.decode(tags)
+		local _tags    = {}
+		-- Make a hash
+		for i,v in ipairs(tags) do _tags[v] = true end
+		-- Delete these from the hash
+		for i = 3,#ARGV do _tags[ARGV[i]] = nil end
+		-- Back into a list
+		local results = {}
+		for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
+		-- json encode them, set, and return
+		tags = cjson.encode(results)
+		redis.call('hset', 'ql:r:' .. jid, 'tags', tags)
+		return tags
+	else
+		return false
+	end
+else
+	error('Recur(): First argument must be one of [on, off, get, update, tag, untag]. Got ' .. tostring(ARGV[1]))
+end

data/lib/qless/qless-core/retry.lua

@@ -0,0 +1,73 @@
+-- retry(0, jid, queue, worker, now, [delay])
+-- ------------------------------------------
+-- This script accepts jid, queue, worker and delay for
+-- retrying a job. This is similar in functionality to 
+-- `put`, except that this counts against the retries 
+-- a job has for a stage.
+--
+-- If the worker is not the worker with a lock on the job,
+-- then it returns false. If the job is not actually running,
+-- then it returns false. Otherwise, it returns the number
+-- of retries remaining. If the allowed retries have been
+-- exhausted, then it is automatically failed, and a negative
+-- number is returned.
+
+if #KEYS ~= 0 then
+	error('Retry(): Got ' .. #KEYS .. ', expected 0')
+end
+
+local jid      = assert(ARGV[1]               , 'Retry(): Arg "jid" missing')
+local queue    = assert(ARGV[2]               , 'Retry(): Arg "queue" missing')
+local worker   = assert(ARGV[3]               , 'Retry(): Arg "worker" missing')
+local now      = assert(tonumber(ARGV[4])     , 'Retry(): Arg "now" missing')
+local delay    = assert(tonumber(ARGV[5] or 0), 'Retry(): Arg "delay" not a number: ' .. tostring(ARGV[5]))
+
+-- Let's see what the old priority, history and tags were
+local oldqueue, state, retries, oldworker, priority = unpack(redis.call('hmget', 'ql:j:' .. jid, 'queue', 'state', 'retries', 'worker', 'priority'))
+
+-- If this isn't the worker that owns
+if oldworker ~= worker or (state ~= 'running') then
+	return false
+end
+
+-- Remove it from the locks key of the old queue
+redis.call('zrem', 'ql:q:' .. oldqueue .. '-locks', jid)
+
+local remaining = redis.call('hincrby', 'ql:j:' .. jid, 'remaining', -1)
+
+-- Remove this job from the worker that was previously working it
+redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
+
+if remaining < 0 then
+	-- Now remove the instance from the schedule, and work queues for the queue it's in
+	local group = 'failed-retries-' .. queue
+	-- First things first, we should get the history
+	local history = redis.call('hget', 'ql:j:' .. jid, 'history')
+	-- Now, take the element of the history for which our provided worker is the worker, and update 'failed'
+	history = cjson.decode(history or '[]')
+	history[#history]['failed'] = now
+	
+	redis.call('hmset', 'ql:j:' .. jid, 'state', 'failed', 'worker', '',
+		'expires', '', 'history', cjson.encode(history), 'failure', cjson.encode({
+			['group']   = group,
+			['message'] = 'Job exhuasted retries in queue "' .. queue .. '"',
+			['when']    = now,
+			['worker']  = worker
+		}))
+	
+	-- Add this type of failure to the list of failures
+	redis.call('sadd', 'ql:failures', group)
+	-- And add this particular instance to the failed types
+	redis.call('lpush', 'ql:f:' .. group, jid)
+else
+	-- Put it in the queue again with a delay. Like put()
+	if delay > 0 then
+		redis.call('zadd', 'ql:q:' .. queue .. '-scheduled', now + delay, jid)
+		redis.call('hset', 'ql:j:' .. jid, 'state', 'scheduled')
+	else
+		redis.call('zadd', 'ql:q:' .. queue .. '-work', priority - (now / 10000000000), jid)
+		redis.call('hset', 'ql:j:' .. jid, 'state', 'waiting')
+	end
+end
+
+return remaining

data/lib/qless/qless-core/ruby/lib/qless-core.rb

@@ -0,0 +1 @@
+require "qless/core"

data/lib/qless/qless-core/ruby/lib/qless/core.rb

@@ -0,0 +1,13 @@
+require "qless/core/version"
+
+module Qless
+  module Core
+    extend self
+    LUA_SCRIPT_DIR = File.expand_path("../core/lua_scripts", __FILE__)
+
+    def script_contents(name)
+      File.read(File.join(LUA_SCRIPT_DIR, "#{name}.lua"))
+    end
+  end
+end
+

data/lib/qless/qless-core/ruby/lib/qless/core/version.rb

@@ -0,0 +1,5 @@
+module Qless
+  module Core
+    VERSION = "0.0.1"
+  end
+end

data/lib/qless/qless-core/ruby/spec/qless_core_spec.rb

@@ -0,0 +1,13 @@
+require 'minitest/autorun'
+require 'minitest/spec'
+$LOAD_PATH.unshift "lib"
+require 'qless/core'
+
+module Qless
+  describe Core do
+    it "can read the lua files" do
+      Qless::Core.script_contents("put").must_include "Put"
+    end
+  end
+end
+

data/lib/qless/qless-core/stats.lua

@@ -0,0 +1,92 @@
+-- Stats(0, queue, date)
+-- ---------------------
+-- Return the current statistics for a given queue on a given date. The results 
+-- are returned are a JSON blob:
+--
+--
+--	{
+--		# These are unimplemented as of yet
+--		'failed': 3,
+--		'retries': 5,
+--		'wait' : {
+--			'total'    : ...,
+--			'mean'     : ...,
+--			'variance' : ...,
+--			'histogram': [
+--				...
+--			]
+--		}, 'run': {
+--			'total'    : ...,
+--			'mean'     : ...,
+--			'variance' : ...,
+--			'histogram': [
+--				...
+--			]
+--		}
+--	}
+--
+-- The histogram's data points are at the second resolution for the first minute,
+-- the minute resolution for the first hour, the 15-minute resolution for the first
+-- day, the hour resolution for the first 3 days, and then at the day resolution
+-- from there on out. The `histogram` key is a list of those values.
+--
+-- Args:
+--    1) queue
+--    2) time
+
+if #KEYS > 0 then error('Stats(): No Keys should be provided') end
+
+local queue = assert(ARGV[1]          , 'Stats(): Arg "queue" missing')
+local time  = assert(tonumber(ARGV[2]), 'Stats(): Arg "time" missing or not a number: ' .. (ARGV[2] or 'nil'))
+
+-- The bin is midnight of the provided day
+-- 24 * 60 * 60 = 86400
+local bin = time - (time % 86400)
+
+-- This a table of all the keys we want to use in order to produce a histogram
+local histokeys = {
+	's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59',
+	'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59',
+	'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23',
+	'd1','d2','d3','d4','d5','d6'
+}
+
+local mkstats = function(name, bin, queue)
+	-- The results we'll be sending back
+	local results = {}
+	
+	local count, mean, vk = unpack(redis.call('hmget', 'ql:s:' .. name .. ':' .. bin .. ':' .. queue, 'total', 'mean', 'vk'))
+	
+	count = tonumber(count) or 0
+	mean  = tonumber(mean) or 0
+	vk    = tonumber(vk)
+	
+	results.count     = count or 0
+	results.mean      = mean  or 0
+	results.histogram = {}
+
+	if not count then
+		results.std = 0
+	else
+		if count > 1 then
+			results.std = math.sqrt(vk / (count - 1))
+		else
+			results.std = 0
+		end
+	end
+
+	local histogram = redis.call('hmget', 'ql:s:' .. name .. ':' .. bin .. ':' .. queue, unpack(histokeys))
+	for i=1,#histokeys do
+		table.insert(results.histogram, tonumber(histogram[i]) or 0)
+	end
+	return results
+end
+
+local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. queue, 'retries', 'failed', 'failures'))
+return cjson.encode({
+	retries  = tonumber(retries  or 0),
+	failed   = tonumber(failed   or 0),
+	failures = tonumber(failures or 0),
+	wait     = mkstats('wait', bin, queue),
+	run      = mkstats('run' , bin, queue)
+})