qless 0.9.3 → 0.10.0

data/lib/qless/qless-core/queues.lua

@@ -1,58 +0,0 @@
--- Queues(0, now, [queue])
--- -----------------------
--- 
--- Return all the queues we know about, with how many jobs are scheduled, waiting,
--- and running in that queue. If a queue name is provided, then only the appropriate
--- response hash should be returned. The response is JSON:
--- 
--- 	[
--- 		{
--- 			'name': 'testing',
--- 			'stalled': 2,
--- 			'waiting': 5,
--- 			'running': 5,
--- 			'scheduled': 10,
--- 			'depends': 5,
--- 			'recurring': 0
--- 		}, {
--- 			...
--- 		}
--- 	]
-
-if #KEYS > 0 then
-	error('Queues(): Got '.. #KEYS .. ' expected 0 KEYS arguments')
-end
-
-local now   = assert(tonumber(ARGV[1]), 'Queues(): Arg "now" missing or not a number: ' .. (ARGV[1] or 'nil'))
-local queue = ARGV[2]
-
-local response = {}
-local queuenames = redis.call('zrange', 'ql:queues', 0, -1)
-
-if queue then
-	local stalled = redis.call('zcount', 'ql:q:' .. queue .. '-locks', 0, now)
-	response = {
-		name      = queue,
-		waiting   = redis.call('zcard', 'ql:q:' .. queue .. '-work'),
-		stalled   = stalled,
-		running   = redis.call('zcard', 'ql:q:' .. queue .. '-locks') - stalled,
-		scheduled = redis.call('zcard', 'ql:q:' .. queue .. '-scheduled'),
-		depends   = redis.call('zcard', 'ql:q:' .. queue .. '-depends'),
-		recurring = redis.call('zcard', 'ql:q:' .. queue .. '-recur')
-	}
-else
-	for index, qname in ipairs(queuenames) do
-		local stalled = redis.call('zcount', 'ql:q:' .. qname .. '-locks', 0, now)
-		table.insert(response, {
-			name      = qname,
-			waiting   = redis.call('zcard', 'ql:q:' .. qname .. '-work'),
-			stalled   = stalled,
-			running   = redis.call('zcard', 'ql:q:' .. qname .. '-locks') - stalled,
-			scheduled = redis.call('zcard', 'ql:q:' .. qname .. '-scheduled'),
-			depends   = redis.call('zcard', 'ql:q:' .. qname .. '-depends'),
-			recurring = redis.call('zcard', 'ql:q:' .. qname .. '-recur')
-		})
-	end
-end
-
-return cjson.encode(response)

data/lib/qless/qless-core/recur.lua

@@ -1,190 +0,0 @@
--- Recur(0, 'on', queue, jid, klass, data, now, 'interval', second, offset, [priority p], [tags t], [retries r])
--- Recur(0, 'off', jid)
--- Recur(0, 'get', jid)
--- Recur(0, 'update', jid, ['priority', priority], ['interval', interval], ['retries', retries], ['data', data], ['klass', klass], ['queue', queue])
--- Recur(0, 'tag', jid, tag, [tag, [...]])
--- Recur(0, 'untag', jid, tag, [tag, [...]])
--- -------------------------------------------------------------------------------------------------------
--- This script takes the name of a queue, and then the info
--- info about the work item, and makes sure that jobs matching
--- its criteria are regularly made available.
-
-if #KEYS ~= 0 then
-	error('Recur(): Got ' .. #KEYS .. ', expected 0 KEYS arguments')
-end
-
-local command  = assert(ARGV[1]               , 'Recur(): Missing first argument')
-
-if command == 'on' then
-	local queue    = assert(ARGV[2]               , 'Recur(): Arg "queue" missing')
-	local jid      = assert(ARGV[3]               , 'Recur(): Arg "jid" missing')
-	local klass    = assert(ARGV[4]               , 'Recur(): Arg "klass" missing')
-	local data     = assert(cjson.decode(ARGV[5]) , 'Recur(): Arg "data" missing or not JSON: '    .. tostring(ARGV[5]))
-	local now      = assert(tonumber(ARGV[6])     , 'Recur(): Arg "now" missing or not a number: ' .. tostring(ARGV[6]))
-	local spec     = assert(ARGV[7]               , 'Recur(): Arg "schedule type" missing')
-	if spec == 'interval' then
-		local interval = assert(tonumber(ARGV[8]) , 'Recur(): Arg "interval" must be a number: '   .. tostring(ARGV[8]))
-		local offset   = assert(tonumber(ARGV[9]) , 'Recur(): Arg "offset" must be a number: '     .. tostring(ARGV[9]))
-		if interval <= 0 then
-			error('Recur(): Arg "interval" must be greater than or equal to 0')
-		end
-		-- Read in all the optional parameters
-		local options = {}
-		for i = 10, #ARGV, 2 do options[ARGV[i]] = ARGV[i + 1] end
-		options.tags     = assert(cjson.decode(options.tags or {}), 'Recur(): Arg "tags" must be JSON-encoded array of string. Got: ' .. tostring(options.tags))
-		options.priority = assert(tonumber(options.priority or 0) , 'Recur(): Arg "priority" must be a number. Got: ' .. tostring(options.priority))
-		options.retries  = assert(tonumber(options.retries  or 0) , 'Recur(): Arg "retries" must be a number. Got: ' .. tostring(options.retries))
-
-		local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
-		count = count or 0
-
-		-- If it has previously been in another queue, then we should remove 
-		-- some information about it
-		if old_queue then
-			redis.call('zrem', 'ql:q:' .. old_queue .. '-recur', jid)
-		end
-		
-		-- Do some insertions
-		redis.call('hmset', 'ql:r:' .. jid,
-			'jid'     , jid,
-			'klass'   , klass,
-			'data'    , cjson.encode(data),
-			'priority', options.priority,
-			'tags'    , cjson.encode(options.tags or {}),
-			'state'   , 'recur',
-			'queue'   , queue,
-			'type'    , 'interval',
-			-- How many jobs we've spawned from this
-			'count'   , count,
-			'interval', interval,
-			'retries' , options.retries)
-		-- Now, we should schedule the next run of the job
-		redis.call('zadd', 'ql:q:' .. queue .. '-recur', now + offset, jid)
-		
-		-- Lastly, we're going to make sure that this item is in the
-		-- set of known queues. We should keep this sorted by the 
-		-- order in which we saw each of these queues
-		if redis.call('zscore', 'ql:queues', queue) == false then
-			redis.call('zadd', 'ql:queues', now, queue)
-		end
-		
-		return jid
-	else
-		error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
-	end
-elseif command == 'off' then
-	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
-	-- First, find out what queue it was attached to
-	local queue = redis.call('hget', 'ql:r:' .. jid, 'queue')
-	if queue then
-		-- Now, delete it from the queue it was attached to, and delete the thing itself
-		redis.call('zrem', 'ql:q:' .. queue .. '-recur', jid)
-		redis.call('del', 'ql:r:' .. jid)
-		return true
-	else
-		return true
-	end
-elseif command == 'get' then
-	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
-	local job = redis.call(
-	    'hmget', 'ql:r:' .. jid, 'jid', 'klass', 'state', 'queue',
-		'priority', 'interval', 'retries', 'count', 'data', 'tags')
-	
-	if not job[1] then
-		return false
-	end
-	
-	return cjson.encode({
-	    jid          = job[1],
-		klass        = job[2],
-	    state        = job[3],
-	    queue        = job[4],
-		priority     = tonumber(job[5]),
-		interval     = tonumber(job[6]),
-		retries      = tonumber(job[7]),
-		count        = tonumber(job[8]),
-		data         = cjson.decode(job[9]),
-		tags         = cjson.decode(job[10])
-	})
-elseif command == 'update' then
-	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
-	local options = {}
-	
-	-- Make sure that the job exists
-	if redis.call('exists', 'ql:r:' .. jid) ~= 0 then
-		for i = 3, #ARGV, 2 do
-			local key = ARGV[i]
-			local value = ARGV[i+1]
-			if key == 'priority' or key == 'interval' or key == 'retries' then
-				value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value))
-				-- If the command is 'interval', then we need to update the time
-				-- when it should next be scheduled
-				if key == 'interval' then
-					local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. jid, 'queue', 'interval'))
-					redis.call('zincrby', 'ql:q:' .. queue .. '-recur', value - tonumber(interval), jid)
-				end
-				redis.call('hset', 'ql:r:' .. jid, key, value)
-			elseif key == 'data' then
-				value = assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value))
-				redis.call('hset', 'ql:r:' .. jid, 'data', cjson.encode(value))
-			elseif key == 'klass' then
-				redis.call('hset', 'ql:r:' .. jid, 'klass', value)
-			elseif key == 'queue' then
-				local queue = redis.call('hget', 'ql:r:' .. jid, 'queue')
-				local score = redis.call('zscore', 'ql:q:' .. queue .. '-recur', jid)
-				redis.call('zrem', 'ql:q:' .. queue .. '-recur', jid)
-				redis.call('zadd', 'ql:q:' .. value .. '-recur', score, jid)
-				redis.call('hset', 'ql:r:' .. jid, 'queue', value)
-			else
-				error('Recur(): Unrecognized option "' .. key .. '"')
-			end
-		end
-		return true
-	else
-		return false
-	end
-elseif command == 'tag' then
-	local jid = assert(ARGV[2], 'Recur(): Arg "jid" missing')
-	local tags = redis.call('hget', 'ql:r:' .. jid, 'tags')
-	-- If the job has been canceled / deleted, then return false
-	if tags then
-		-- Decode the json blob, convert to dictionary
-		tags = cjson.decode(tags)
-		local _tags = {}
-		for i,v in ipairs(tags) do _tags[v] = true end
-		
-		-- Otherwise, add the job to the sorted set with that tags
-		for i=3,#ARGV do if _tags[ARGV[i]] == nil then table.insert(tags, ARGV[i]) end end
-		
-		tags = cjson.encode(tags)
-		redis.call('hset', 'ql:r:' .. jid, 'tags', tags)
-		return tags
-	else
-		return false
-	end
-elseif command == 'untag' then
-	local jid  = assert(ARGV[2], 'Recur(): Arg "jid" missing')
-	-- Get the existing tags
-	local tags = redis.call('hget', 'ql:r:' .. jid, 'tags')
-	-- If the job has been canceled / deleted, then return false
-	if tags then
-		-- Decode the json blob, convert to dictionary
-		tags = cjson.decode(tags)
-		local _tags    = {}
-		-- Make a hash
-		for i,v in ipairs(tags) do _tags[v] = true end
-		-- Delete these from the hash
-		for i = 3,#ARGV do _tags[ARGV[i]] = nil end
-		-- Back into a list
-		local results = {}
-		for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
-		-- json encode them, set, and return
-		tags = cjson.encode(results)
-		redis.call('hset', 'ql:r:' .. jid, 'tags', tags)
-		return tags
-	else
-		return false
-	end
-else
-	error('Recur(): First argument must be one of [on, off, get, update, tag, untag]. Got ' .. tostring(ARGV[1]))
-end

data/lib/qless/qless-core/retry.lua

@@ -1,73 +0,0 @@
--- retry(0, jid, queue, worker, now, [delay])
--- ------------------------------------------
--- This script accepts jid, queue, worker and delay for
--- retrying a job. This is similar in functionality to 
--- `put`, except that this counts against the retries 
--- a job has for a stage.
---
--- If the worker is not the worker with a lock on the job,
--- then it returns false. If the job is not actually running,
--- then it returns false. Otherwise, it returns the number
--- of retries remaining. If the allowed retries have been
--- exhausted, then it is automatically failed, and a negative
--- number is returned.
-
-if #KEYS ~= 0 then
-	error('Retry(): Got ' .. #KEYS .. ', expected 0')
-end
-
-local jid      = assert(ARGV[1]               , 'Retry(): Arg "jid" missing')
-local queue    = assert(ARGV[2]               , 'Retry(): Arg "queue" missing')
-local worker   = assert(ARGV[3]               , 'Retry(): Arg "worker" missing')
-local now      = assert(tonumber(ARGV[4])     , 'Retry(): Arg "now" missing')
-local delay    = assert(tonumber(ARGV[5] or 0), 'Retry(): Arg "delay" not a number: ' .. tostring(ARGV[5]))
-
--- Let's see what the old priority, history and tags were
-local oldqueue, state, retries, oldworker, priority = unpack(redis.call('hmget', 'ql:j:' .. jid, 'queue', 'state', 'retries', 'worker', 'priority'))
-
--- If this isn't the worker that owns
-if oldworker ~= worker or (state ~= 'running') then
-	return false
-end
-
--- Remove it from the locks key of the old queue
-redis.call('zrem', 'ql:q:' .. oldqueue .. '-locks', jid)
-
-local remaining = redis.call('hincrby', 'ql:j:' .. jid, 'remaining', -1)
-
--- Remove this job from the worker that was previously working it
-redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
-
-if remaining < 0 then
-	-- Now remove the instance from the schedule, and work queues for the queue it's in
-	local group = 'failed-retries-' .. queue
-	-- First things first, we should get the history
-	local history = redis.call('hget', 'ql:j:' .. jid, 'history')
-	-- Now, take the element of the history for which our provided worker is the worker, and update 'failed'
-	history = cjson.decode(history or '[]')
-	history[#history]['failed'] = now
-	
-	redis.call('hmset', 'ql:j:' .. jid, 'state', 'failed', 'worker', '',
-		'expires', '', 'history', cjson.encode(history), 'failure', cjson.encode({
-			['group']   = group,
-			['message'] = 'Job exhausted retries in queue "' .. queue .. '"',
-			['when']    = now,
-			['worker']  = worker
-		}))
-	
-	-- Add this type of failure to the list of failures
-	redis.call('sadd', 'ql:failures', group)
-	-- And add this particular instance to the failed types
-	redis.call('lpush', 'ql:f:' .. group, jid)
-else
-	-- Put it in the queue again with a delay. Like put()
-	if delay > 0 then
-		redis.call('zadd', 'ql:q:' .. queue .. '-scheduled', now + delay, jid)
-		redis.call('hset', 'ql:j:' .. jid, 'state', 'scheduled')
-	else
-		redis.call('zadd', 'ql:q:' .. queue .. '-work', priority - (now / 10000000000), jid)
-		redis.call('hset', 'ql:j:' .. jid, 'state', 'waiting')
-	end
-end
-
-return remaining

data/lib/qless/qless-core/stats.lua

@@ -1,92 +0,0 @@
--- Stats(0, queue, date)
--- ---------------------
--- Return the current statistics for a given queue on a given date. The results 
--- are returned are a JSON blob:
---
---
---	{
---		# These are unimplemented as of yet
---		'failed': 3,
---		'retries': 5,
---		'wait' : {
---			'total'    : ...,
---			'mean'     : ...,
---			'variance' : ...,
---			'histogram': [
---				...
---			]
---		}, 'run': {
---			'total'    : ...,
---			'mean'     : ...,
---			'variance' : ...,
---			'histogram': [
---				...
---			]
---		}
---	}
---
--- The histogram's data points are at the second resolution for the first minute,
--- the minute resolution for the first hour, the 15-minute resolution for the first
--- day, the hour resolution for the first 3 days, and then at the day resolution
--- from there on out. The `histogram` key is a list of those values.
---
--- Args:
---    1) queue
---    2) time
-
-if #KEYS > 0 then error('Stats(): No Keys should be provided') end
-
-local queue = assert(ARGV[1]          , 'Stats(): Arg "queue" missing')
-local time  = assert(tonumber(ARGV[2]), 'Stats(): Arg "time" missing or not a number: ' .. (ARGV[2] or 'nil'))
-
--- The bin is midnight of the provided day
--- 24 * 60 * 60 = 86400
-local bin = time - (time % 86400)
-
--- This a table of all the keys we want to use in order to produce a histogram
-local histokeys = {
-	's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59',
-	'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59',
-	'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23',
-	'd1','d2','d3','d4','d5','d6'
-}
-
-local mkstats = function(name, bin, queue)
-	-- The results we'll be sending back
-	local results = {}
-	
-	local count, mean, vk = unpack(redis.call('hmget', 'ql:s:' .. name .. ':' .. bin .. ':' .. queue, 'total', 'mean', 'vk'))
-	
-	count = tonumber(count) or 0
-	mean  = tonumber(mean) or 0
-	vk    = tonumber(vk)
-	
-	results.count     = count or 0
-	results.mean      = mean  or 0
-	results.histogram = {}
-
-	if not count then
-		results.std = 0
-	else
-		if count > 1 then
-			results.std = math.sqrt(vk / (count - 1))
-		else
-			results.std = 0
-		end
-	end
-
-	local histogram = redis.call('hmget', 'ql:s:' .. name .. ':' .. bin .. ':' .. queue, unpack(histokeys))
-	for i=1,#histokeys do
-		table.insert(results.histogram, tonumber(histogram[i]) or 0)
-	end
-	return results
-end
-
-local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. queue, 'retries', 'failed', 'failures'))
-return cjson.encode({
-	retries  = tonumber(retries  or 0),
-	failed   = tonumber(failed   or 0),
-	failures = tonumber(failures or 0),
-	wait     = mkstats('wait', bin, queue),
-	run      = mkstats('run' , bin, queue)
-})

data/lib/qless/qless-core/tag.lua

@@ -1,100 +0,0 @@
--- tag(0, ('add' | 'remove'), jid, now, tag, [tag, ...])
--- tag(0, 'get', tag, [offset, [count]])
--- tag(0, 'top', [offset, [count]])
--- ------------------------------------------------------------------------------------------------------------------
--- Accepts a jid, 'add' or 'remove', and then a list of tags
--- to either add or remove from the job. Alternatively, 'get',
--- a tag to get jobs associated with that tag, and offset and
--- count
---
--- If 'add' or 'remove', the response is a list of the jobs
--- current tags, or False if the job doesn't exist. If 'get',
--- the response is of the form:
---
---	{
---		total: ...,
---		jobs: [
---			jid,
---			...
---		]
---	}
---
--- If 'top' is supplied, it returns the most commonly-used tags
--- in a paginated fashion.
-
-if #KEYS ~= 0 then
-	error('Tag(): Got ' .. #KEYS .. ', expected 0')
-end
-
-local command = assert(ARGV[1], 'Tag(): Missing first arg "add", "remove" or "get"')
-
-if command == 'add' then
-	local jid  = assert(ARGV[2]          , 'Tag(): Arg "jid" missing')
-	local now  = assert(tonumber(ARGV[3]), 'Tag(): Arg "now" is not a number')
-	local tags = redis.call('hget', 'ql:j:' .. jid, 'tags')
-	-- If the job has been canceled / deleted, then return false
-	if tags then
-		-- Decode the json blob, convert to dictionary
-		tags = cjson.decode(tags)
-		local _tags = {}
-		for i,v in ipairs(tags) do _tags[v] = true end
-	
-		-- Otherwise, add the job to the sorted set with that tags
-		for i=4,#ARGV do
-			local tag = ARGV[i]
-			if _tags[tag] == nil then
-				table.insert(tags, tag)
-			end
-			redis.call('zadd', 'ql:t:' .. tag, now, jid)
-			redis.call('zincrby', 'ql:tags', 1, tag)
-		end
-	
-		tags = cjson.encode(tags)
-		redis.call('hset', 'ql:j:' .. jid, 'tags', tags)
-		return tags
-	else
-		return false
-	end
-elseif command == 'remove' then
-	local jid  = assert(ARGV[2]          , 'Tag(): Arg "jid" missing')
-	local now  = assert(tonumber(ARGV[3]), 'Tag(): Arg "now" is not a number')
-	local tags = redis.call('hget', 'ql:j:' .. jid, 'tags')
-	-- If the job has been canceled / deleted, then return false
-	if tags then
-		-- Decode the json blob, convert to dictionary
-		tags = cjson.decode(tags)
-		local _tags = {}
-		for i,v in ipairs(tags) do _tags[v] = true end
-	
-		-- Otherwise, add the job to the sorted set with that tags
-		for i=4,#ARGV do
-			local tag = ARGV[i]
-			_tags[tag] = nil
-			redis.call('zrem', 'ql:t:' .. tag, jid)
-			redis.call('zincrby', 'ql:tags', -1, tag)
-		end
-	
-		local results = {}
-		for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
-	
-		tags = cjson.encode(results)
-		redis.call('hset', 'ql:j:' .. jid, 'tags', tags)
-		return tags
-	else
-		return false
-	end
-elseif command == 'get' then
-	local tag    = assert(ARGV[2]                , 'Tag(): Arg "tag" missing')
-	local offset = assert(tonumber(ARGV[3] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(ARGV[3]))
-	local count  = assert(tonumber(ARGV[4] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(ARGV[4]))
-	return cjson.encode({
-		total = redis.call('zcard', 'ql:t:' .. tag),
-		jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, count)
-	})
-elseif command == 'top' then
-	local offset = assert(tonumber(ARGV[2] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(ARGV[2]))
-	local count  = assert(tonumber(ARGV[3] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(ARGV[3]))
-	return cjson.encode(redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count))
-else
-	error('Tag(): First argument must be "add", "remove" or "get"')
-end