qless 0.9.3 → 0.10.0

data/lib/qless/qless-core/config.lua

@@ -1,56 +0,0 @@
--- config(0, 'get', [option])
--- config(0, 'set', option, value)
--- config(0, 'unset', option)
--- -------------------------------------------------------------------------------------------------------
--- This script provides an interface to get, set, and unset configuration
--- options.
---
--- Args:
---    1) [option]
-
-if #KEYS > 0 then error('Config(): No keys should be provided') end
-
-local command = ARGV[1]
-
-local defaults = {
-	['application']        = 'qless',
-	['heartbeat']          = 60,
-	['stats-history']      = 30,
-	['histogram-history']  = 7,
-	['jobs-history-count'] = 50000,
-	['jobs-history']       = 604800
-}
-
-if command == 'get' then
-	if ARGV[2] then
-		return redis.call('hget', 'ql:config', ARGV[2]) or defaults[ARGV[2]]
-	else
-		-- Inspired by redis-lua https://github.com/nrk/redis-lua/blob/version-2.0/src/redis.lua
-		local reply = redis.call('hgetall', 'ql:config')
-	    for i = 1, #reply, 2 do defaults[reply[i]] = reply[i + 1] end
-		return cjson.encode(defaults)
-	end
-elseif command == 'set' then
-	local option = assert(ARGV[2], 'Config(): Arg "option" missing')
-	local value  = assert(ARGV[3], 'Config(): Arg "value" missing')
-	-- Send out a log message
-	redis.call('publish', 'ql:log', cjson.encode({
-		event  = 'config_set',
-		option = option
-	}))
-
-	redis.call('hset', 'ql:config', option, value)
-elseif command == 'unset' then
-	local option = assert(ARGV[2], 'Config(): Arg "option" missing')
-	-- Send out a log message
-	redis.call('publish', 'ql:log', cjson.encode({
-		event  = 'config_unset',
-		option = option
-	}))
-
-	redis.call('hdel', 'ql:config', option)
-else
-	error('Config(): Unrecognized command ' .. command)
-end
-
-

data/lib/qless/qless-core/depends.lua

@@ -1,65 +0,0 @@
--- Depends(0, jid, ('on', [jid, [jid, [...]]]) | ('off', ('all' | [jid, [jid, [...]]]))
--- ------------------------------------------------------------------------------------
--- Add or remove dependencies a job has. If 'on' is provided, the provided jids are 
--- added as dependencies. If 'off' and 'all' are provided, then all the current dependencies
--- are removed. If 'off' is provided and the next argument is not 'all', then those
--- jids are removed as dependencies.
---
--- If a job is not already in the 'depends' state, then this call will return false.
--- Otherwise, it will return true
---
--- Args:
---    1) jid
-
-if #KEYS > 0 then error('Depends(): No Keys should be provided') end
-
-local jid     = assert(ARGV[1], 'Depends(): Arg "jid" missing.')
-local command = assert(ARGV[2], 'Depends(): Arg 2 missing')
-
-if redis.call('hget', 'ql:j:' .. jid, 'state') ~= 'depends' then
-	return false
-end
-
-if ARGV[2] == 'on' then
-	-- These are the jids we legitimately have to wait on
-	for i=3,#ARGV do
-		local j = ARGV[i]
-		-- Make sure it's something other than 'nil' or complete.
-		local state = redis.call('hget', 'ql:j:' .. j, 'state')
-		if (state and state ~= 'complete') then
-			redis.call('sadd', 'ql:j:' .. j .. '-dependents'  , jid)
-			redis.call('sadd', 'ql:j:' .. jid .. '-dependencies', j)
-		end
-	end
-	return true
-elseif ARGV[2] == 'off' then
-	if ARGV[3] == 'all' then
-		for i, j in ipairs(redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')) do
-			redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
-		end
-		redis.call('del', 'ql:j:' .. jid .. '-dependencies')
-		local q, p = unpack(redis.call('hmget', 'ql:j:' .. jid, 'queue', 'priority'))
-		if q then
-			redis.call('zrem', 'ql:q:' .. q .. '-depends', jid)
-			redis.call('zadd', 'ql:q:' .. q .. '-work', p, jid)
-			redis.call('hset', 'ql:j:' .. jid, 'state', 'waiting')
-		end
-	else
-		for i=3,#ARGV do
-			local j = ARGV[i]
-			redis.call('srem', 'ql:j:' .. j .. '-dependents', jid)
-			redis.call('srem', 'ql:j:' .. jid .. '-dependencies', j)
-			if redis.call('scard', 'ql:j:' .. jid .. '-dependencies') == 0 then
-				local q, p = unpack(redis.call('hmget', 'ql:j:' .. jid, 'queue', 'priority'))
-				if q then
-					redis.call('zrem', 'ql:q:' .. q .. '-depends', jid)
-					redis.call('zadd', 'ql:q:' .. q .. '-work', p, jid)
-					redis.call('hset', 'ql:j:' .. jid, 'state', 'waiting')
-				end
-			end
-		end
-	end
-	return true
-else
-	error('Depends(): Second arg must be "on" or "off"')
-end

data/lib/qless/qless-core/deregister_workers.lua

@@ -1,12 +0,0 @@
--- DeregisterWorkers(0, worker)
--- This script takes the name of a worker(s) on removes it/them 
--- from the ql:workers set.
---
--- Args: The list of workers to deregister.
-
-if #KEYS > 0 then error('DeregisterWorkers(): No Keys should be provided') end
-if #ARGV < 1 then error('DeregisterWorkers(): Must provide at least one worker to deregister') end
-
-local key = 'ql:workers'
-
-redis.call('zrem', key, unpack(ARGV))

data/lib/qless/qless-core/fail.lua

@@ -1,117 +0,0 @@
--- Fail(0, jid, worker, group, message, now, [data])
--- -------------------------------------------------
--- Mark the particular job as failed, with the provided group, and a more
--- specific message. By `group`, we mean some phrase that might be one of
--- several categorical modes of failure. The `message` is something more
--- job-specific, like perhaps a traceback.
--- 
--- This method should __not__ be used to note that a job has been dropped or
--- has failed in a transient way. This method __should__ be used to note that
--- a job has something really wrong with it that must be remedied.
--- 
--- The motivation behind the `group` is so that similar errors can be grouped
--- together. Optionally, updated data can be provided for the job. A job in
--- any state can be marked as failed. If it has been given to a worker as a 
--- job, then its subsequent requests to heartbeat or complete that job will
--- fail. Failed jobs are kept until they are canceled or completed.
---
--- __Returns__ the id of the failed job if successful, or `False` on failure.
---
--- Args:
---    1) jid
---    2) worker
---    3) group
---    4) message
---    5) the current time
---    6) [data]
-
-if #KEYS > 0 then error('Fail(): No Keys should be provided') end
-
-local jid     = assert(ARGV[1]          , 'Fail(): Arg "jid" missing')
-local worker  = assert(ARGV[2]          , 'Fail(): Arg "worker" missing')
-local group   = assert(ARGV[3]          , 'Fail(): Arg "group" missing')
-local message = assert(ARGV[4]          , 'Fail(): Arg "message" missing')
-local now     = assert(tonumber(ARGV[5]), 'Fail(): Arg "now" missing or malformed: ' .. (ARGV[5] or 'nil'))
-local data    = ARGV[6]
-
--- The bin is midnight of the provided day
--- 24 * 60 * 60 = 86400
-local bin = now - (now % 86400)
-
-if data then
-	data = cjson.decode(data)
-end
-
--- First things first, we should get the history
-local history, queue, state = unpack(redis.call('hmget', 'ql:j:' .. jid, 'history', 'queue', 'state'))
-
--- If the job has been completed, we cannot fail it
-if state ~= 'running' then
-	return false
-end
-
--- Send out a log message
-redis.call('publish', 'ql:log', cjson.encode({
-	jid     = jid,
-	event   = 'failed',
-	worker  = worker,
-	group   = group,
-	message = message
-}))
-
-if redis.call('zscore', 'ql:tracked', jid) ~= false then
-	redis.call('publish', 'failed', jid)
-end
-
--- Remove this job from the jobs that the worker that was running it has
-redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
-
--- Now, take the element of the history for which our provided worker is the worker, and update 'failed'
-history = cjson.decode(history or '[]')
-if #history > 0 then
-	for i=#history,1,-1 do
-		if history[i]['worker'] == worker then
-			history[i]['failed'] = math.floor(now)
-		end
-	end
-else
-	history = {
-		{
-			worker = worker,
-			failed = math.floor(now)
-		}
-	}
-end
-
--- Increment the number of failures for that queue for the
--- given day.
-redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1)
-redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed'  , 1)
-
--- Now remove the instance from the schedule, and work queues for the queue it's in
-redis.call('zrem', 'ql:q:' .. queue .. '-work', jid)
-redis.call('zrem', 'ql:q:' .. queue .. '-locks', jid)
-redis.call('zrem', 'ql:q:' .. queue .. '-scheduled', jid)
-
--- The reason that this appears here is that the above will fail if the job doesn't exist
-if data then
-	redis.call('hset', 'ql:j:' .. jid, 'data', cjson.encode(data))
-end
-
-redis.call('hmset', 'ql:j:' .. jid, 'state', 'failed', 'worker', '',
-	'expires', '', 'history', cjson.encode(history), 'failure', cjson.encode({
-		['group']   = group,
-		['message'] = message,
-		['when']    = math.floor(now),
-		['worker']  = worker
-	}))
-
--- Add this group of failure to the list of failures
-redis.call('sadd', 'ql:failures', group)
--- And add this particular instance to the failed groups
-redis.call('lpush', 'ql:f:' .. group, jid)
-
--- Here is where we'd intcrement stats about the particular stage
--- and possibly the workers
-
-return jid

data/lib/qless/qless-core/failed.lua

@@ -1,83 +0,0 @@
--- Failed(0, [group, [start, [limit]]])
--- ------------------------------------
--- If no group is provided, this returns a JSON blob of the counts of the various
--- groups of failures known. If a group is provided, it will report up to `limit`
--- from `start` of the jobs affected by that issue. __Returns__ a JSON blob.
--- 
--- 	# If no group, then...
--- 	{
--- 		'group1': 1,
--- 		'group2': 5,
--- 		...
--- 	}
--- 	
--- 	# If a group is provided, then...
--- 	{
---      'total': 20,
--- 		'jobs': [
--- 			{
--- 				# All the normal keys for a job
--- 				'jid': ...,
--- 				'data': ...
--- 				# The message for this particular instance
--- 				'message': ...,
--- 				'group': ...,
--- 			}, ...
--- 		]
--- 	}
---
--- Args:
---    1) [group]
---    2) [start]
---    3) [limit]
-
-if #KEYS > 0 then error('Failed(): No Keys should be provided') end
-
-local group = ARGV[1]
-local start = assert(tonumber(ARGV[2] or  0), 'Failed(): Arg "start" is not a number: ' .. (ARGV[2] or 'nil'))
-local limit = assert(tonumber(ARGV[3] or 25), 'Failed(): Arg "limit" is not a number: ' .. (ARGV[3] or 'nil'))
-
-if group then
-	-- If a group was provided, then we should do paginated lookup into that
-	local response = {
-		total = redis.call('llen', 'ql:f:' .. group),
-		jobs  = {}
-	}
-	local jids = redis.call('lrange', 'ql:f:' .. group, start, limit - 1)
-	for index, jid in ipairs(jids) do
-		local job = redis.call(
-		    'hmget', 'ql:j:' .. jid, 'jid', 'klass', 'state', 'queue', 'worker', 'priority',
-			'expires', 'retries', 'remaining', 'data', 'tags', 'history', 'failure')
-		
-		table.insert(response.jobs, {
-		    jid          = job[1],
-			klass        = job[2],
-		    state        = job[3],
-		    queue        = job[4],
-			worker       = job[5] or '',
-			tracked      = redis.call('zscore', 'ql:tracked', jid) ~= false,
-			priority     = tonumber(job[6]),
-			expires      = tonumber(job[7]) or 0,
-			retries      = tonumber(job[8]),
-			remaining    = tonumber(job[9]),
-			data         = cjson.decode(job[10]),
-			tags         = cjson.decode(job[11]),
-		    history      = cjson.decode(job[12]),
-			failure      = cjson.decode(job[13] or '{}'),
-			dependents   = redis.call('smembers', 'ql:j:' .. jid .. '-dependents'),
-			-- A job in the failed state can not have dependencies
-			-- because it has been popped off of a queue, which 
-			-- means all of its dependencies have been satisfied
-			dependencies = {}
-		})
-	end
-	return cjson.encode(response)
-else
-	-- Otherwise, we should just list all the known failure groups we have
-	local response = {}
-	local groups = redis.call('smembers', 'ql:failures')
-	for index, group in ipairs(groups) do
-		response[group] = redis.call('llen', 'ql:f:' .. group)
-	end
-	return cjson.encode(response)
-end

data/lib/qless/qless-core/get.lua

@@ -1,37 +0,0 @@
--- This gets all the data associated with the job with the
--- provided id.
---
--- Args:
---    1) jid
-
-if #KEYS > 0 then error('Get(): No Keys should be provided') end
-
-local jid = assert(ARGV[1], 'Get(): Arg "jid" missing')
-
--- Let's get all the data we can
-local job = redis.call(
-    'hmget', 'ql:j:' .. jid, 'jid', 'klass', 'state', 'queue', 'worker', 'priority',
-	'expires', 'retries', 'remaining', 'data', 'tags', 'history', 'failure')
-
-if not job[1] then
-	return false
-end
-		
-return cjson.encode({
-    jid          = job[1],
-	klass        = job[2],
-    state        = job[3],
-    queue        = job[4],
-	worker       = job[5] or '',
-	tracked      = redis.call('zscore', 'ql:tracked', jid) ~= false,
-	priority     = tonumber(job[6]),
-	expires      = tonumber(job[7]) or 0,
-	retries      = tonumber(job[8]),
-	remaining    = tonumber(job[9]),
-	data         = cjson.decode(job[10]),
-	tags         = cjson.decode(job[11]),
-    history      = cjson.decode(job[12]),
-	failure      = cjson.decode(job[13] or '{}'),
-	dependents   = redis.call('smembers', 'ql:j:' .. jid .. '-dependents'),
-	dependencies = redis.call('smembers', 'ql:j:' .. jid .. '-dependencies')
-})

data/lib/qless/qless-core/heartbeat.lua

@@ -1,51 +0,0 @@
--- This scripts conducts a heartbeat for a job, and returns
--- either the new expiration or False if the lock has been
--- given to another node
---
--- Args:
---    1) jid
---    2) worker
---    3) now
---    4) [data]
-
-if #KEYS > 0 then error('Heartbeat(): No Keys should be provided') end
-
-local jid    = assert(ARGV[1]          , 'Heartbeat(): Arg "jid" missing')
-local worker = assert(ARGV[2]          , 'Heartbeat(): Arg "worker" missing')
-local now    = assert(tonumber(ARGV[3]), 'Heartbeat(): Arg "now" missing')
-local data   = ARGV[4]
-
--- We should find the heartbeat interval for this queue
--- heartbeat. First, though, we need to find the queue
--- this particular job is in
-local queue     = redis.call('hget', 'ql:j:' .. jid, 'queue') or ''
-local _hb, _qhb = unpack(redis.call('hmget', 'ql:config', 'heartbeat', queue .. '-heartbeat'))
-local expires   = now + tonumber(_qhb or _hb or 60)
-
-if data then
-	data = cjson.decode(data)
-end
-
--- First, let's see if the worker still owns this job, and there is a worker
-local job_worker = redis.call('hget', 'ql:j:' .. jid, 'worker')
-if job_worker ~= worker or #job_worker == 0 then
-    return false
-else
-    -- Otherwise, optionally update the user data, and the heartbeat
-    if data then
-        -- I don't know if this is wise, but I'm decoding and encoding
-        -- the user data to hopefully ensure its sanity
-        redis.call('hmset', 'ql:j:' .. jid, 'expires', expires, 'worker', worker, 'data', cjson.encode(data))
-    else
-        redis.call('hmset', 'ql:j:' .. jid, 'expires', expires, 'worker', worker)
-    end
-	
-	-- Update hwen this job was last updated on that worker
-	-- Add this job to the list of jobs handled by this worker
-	redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
-	
-    -- And now we should just update the locks
-    local queue = redis.call('hget', 'ql:j:' .. jid, 'queue')
-    redis.call('zadd', 'ql:q:'.. queue .. '-locks', expires, jid)
-    return expires
-end

data/lib/qless/qless-core/jobs.lua

@@ -1,41 +0,0 @@
--- Jobs(0, 'complete' | (('stalled' | 'running' | 'scheduled' | 'depends', 'recurring'), now, queue) [offset, [count]])
--- -------------------------------------------------------------------------------------------------------
--- 
--- Return all the job ids currently considered to be in the provided state
--- in a particular queue. The response is a list of job ids:
--- 
--- 	[
---		jid1, 
---		jid2,
---		...
---	]
-
-if #KEYS > 0 then
-	error('Jobs(): Got '.. #KEYS .. ' expected 0 KEYS arguments')
-end
-
-local t      = assert(ARGV[1]                , 'Jobs(): Arg "type" missing')
-if t == 'complete' then
-	local offset = assert(tonumber(ARGV[2] or 0) , 'Jobs(): Arg "offset" not a number: ' .. tostring(ARGV[2]))
-	local count  = assert(tonumber(ARGV[3] or 25), 'Jobs(): Arg "count" not a number: ' .. tostring(ARGV[3]))
-	return redis.call('zrevrange', 'ql:completed', offset, offset + count - 1)
-else
-	local now    = assert(tonumber(ARGV[2])      , 'Jobs(): Arg "now" missing or not a number: ' .. tostring(ARGV[2]))
-	local queue  = assert(ARGV[3]                , 'Jobs(): Arg "queue" missing')
-	local offset = assert(tonumber(ARGV[4] or 0) , 'Jobs(): Arg "offset" not a number: ' .. tostring(ARGV[4]))
-	local count  = assert(tonumber(ARGV[5] or 25), 'Jobs(): Arg "count" not a number: ' .. tostring(ARGV[5]))
-
-	if t == 'running' then
-		return redis.call('zrangebyscore', 'ql:q:' .. queue .. '-locks', now, 133389432700, 'limit', offset, count)
-	elseif t == 'stalled' then
-		return redis.call('zrangebyscore', 'ql:q:' .. queue .. '-locks', 0, now, 'limit', offset, count)
-	elseif t == 'scheduled' then
-		return redis.call('zrange', 'ql:q:' .. queue .. '-scheduled', offset, offset + count - 1)
-	elseif t == 'depends' then
-		return redis.call('zrange', 'ql:q:' .. queue .. '-depends', offset, offset + count - 1)
-	elseif t == 'recurring' then
-		return redis.call('zrange', 'ql:q:' .. queue .. '-recur', offset, offset + count - 1)
-	else
-		error('Jobs(): Unknown type "' .. t .. '"')
-	end
-end