qless 0.9.3 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. data/Gemfile +9 -3
  2. data/README.md +70 -25
  3. data/Rakefile +125 -9
  4. data/exe/install_phantomjs +21 -0
  5. data/lib/qless.rb +115 -76
  6. data/lib/qless/config.rb +11 -9
  7. data/lib/qless/failure_formatter.rb +43 -0
  8. data/lib/qless/job.rb +201 -102
  9. data/lib/qless/job_reservers/ordered.rb +7 -1
  10. data/lib/qless/job_reservers/round_robin.rb +16 -6
  11. data/lib/qless/job_reservers/shuffled_round_robin.rb +9 -2
  12. data/lib/qless/lua/qless-lib.lua +2463 -0
  13. data/lib/qless/lua/qless.lua +2012 -0
  14. data/lib/qless/lua_script.rb +63 -12
  15. data/lib/qless/middleware/memory_usage_monitor.rb +62 -0
  16. data/lib/qless/middleware/metriks.rb +45 -0
  17. data/lib/qless/middleware/redis_reconnect.rb +6 -3
  18. data/lib/qless/middleware/requeue_exceptions.rb +94 -0
  19. data/lib/qless/middleware/retry_exceptions.rb +38 -9
  20. data/lib/qless/middleware/sentry.rb +3 -7
  21. data/lib/qless/middleware/timeout.rb +64 -0
  22. data/lib/qless/queue.rb +90 -55
  23. data/lib/qless/server.rb +177 -130
  24. data/lib/qless/server/views/_job.erb +33 -15
  25. data/lib/qless/server/views/completed.erb +11 -0
  26. data/lib/qless/server/views/layout.erb +70 -11
  27. data/lib/qless/server/views/overview.erb +93 -53
  28. data/lib/qless/server/views/queue.erb +9 -8
  29. data/lib/qless/server/views/queues.erb +18 -1
  30. data/lib/qless/subscriber.rb +37 -22
  31. data/lib/qless/tasks.rb +5 -10
  32. data/lib/qless/test_helpers/worker_helpers.rb +55 -0
  33. data/lib/qless/version.rb +3 -1
  34. data/lib/qless/worker.rb +4 -413
  35. data/lib/qless/worker/base.rb +247 -0
  36. data/lib/qless/worker/forking.rb +245 -0
  37. data/lib/qless/worker/serial.rb +41 -0
  38. metadata +135 -52
  39. data/lib/qless/qless-core/cancel.lua +0 -101
  40. data/lib/qless/qless-core/complete.lua +0 -233
  41. data/lib/qless/qless-core/config.lua +0 -56
  42. data/lib/qless/qless-core/depends.lua +0 -65
  43. data/lib/qless/qless-core/deregister_workers.lua +0 -12
  44. data/lib/qless/qless-core/fail.lua +0 -117
  45. data/lib/qless/qless-core/failed.lua +0 -83
  46. data/lib/qless/qless-core/get.lua +0 -37
  47. data/lib/qless/qless-core/heartbeat.lua +0 -51
  48. data/lib/qless/qless-core/jobs.lua +0 -41
  49. data/lib/qless/qless-core/pause.lua +0 -18
  50. data/lib/qless/qless-core/peek.lua +0 -165
  51. data/lib/qless/qless-core/pop.lua +0 -314
  52. data/lib/qless/qless-core/priority.lua +0 -32
  53. data/lib/qless/qless-core/put.lua +0 -169
  54. data/lib/qless/qless-core/qless-lib.lua +0 -2354
  55. data/lib/qless/qless-core/qless.lua +0 -1862
  56. data/lib/qless/qless-core/queues.lua +0 -58
  57. data/lib/qless/qless-core/recur.lua +0 -190
  58. data/lib/qless/qless-core/retry.lua +0 -73
  59. data/lib/qless/qless-core/stats.lua +0 -92
  60. data/lib/qless/qless-core/tag.lua +0 -100
  61. data/lib/qless/qless-core/track.lua +0 -79
  62. data/lib/qless/qless-core/unfail.lua +0 -54
  63. data/lib/qless/qless-core/unpause.lua +0 -12
  64. data/lib/qless/qless-core/workers.lua +0 -69
  65. data/lib/qless/wait_until.rb +0 -19
@@ -0,0 +1,2012 @@
1
+ -- Current SHA: 525c39000dc71df53a3502491cb4daf0e1128f1d
2
+ -- This is a generated file
3
+ local Qless = {
4
+ ns = 'ql:'
5
+ }
6
+
7
+ local QlessQueue = {
8
+ ns = Qless.ns .. 'q:'
9
+ }
10
+ QlessQueue.__index = QlessQueue
11
+
12
+ local QlessWorker = {
13
+ ns = Qless.ns .. 'w:'
14
+ }
15
+ QlessWorker.__index = QlessWorker
16
+
17
+ local QlessJob = {
18
+ ns = Qless.ns .. 'j:'
19
+ }
20
+ QlessJob.__index = QlessJob
21
+
22
+ local QlessRecurringJob = {}
23
+ QlessRecurringJob.__index = QlessRecurringJob
24
+
25
+ Qless.config = {}
26
+
27
+ function table.extend(self, other)
28
+ for i, v in ipairs(other) do
29
+ table.insert(self, v)
30
+ end
31
+ end
32
+
33
+ function Qless.publish(channel, message)
34
+ redis.call('publish', Qless.ns .. channel, message)
35
+ end
36
+
37
+ function Qless.job(jid)
38
+ assert(jid, 'Job(): no jid provided')
39
+ local job = {}
40
+ setmetatable(job, QlessJob)
41
+ job.jid = jid
42
+ return job
43
+ end
44
+
45
+ function Qless.recurring(jid)
46
+ assert(jid, 'Recurring(): no jid provided')
47
+ local job = {}
48
+ setmetatable(job, QlessRecurringJob)
49
+ job.jid = jid
50
+ return job
51
+ end
52
+
53
+ function Qless.failed(group, start, limit)
54
+ start = assert(tonumber(start or 0),
55
+ 'Failed(): Arg "start" is not a number: ' .. (start or 'nil'))
56
+ limit = assert(tonumber(limit or 25),
57
+ 'Failed(): Arg "limit" is not a number: ' .. (limit or 'nil'))
58
+
59
+ if group then
60
+ return {
61
+ total = redis.call('llen', 'ql:f:' .. group),
62
+ jobs = redis.call('lrange', 'ql:f:' .. group, start, start + limit - 1)
63
+ }
64
+ else
65
+ local response = {}
66
+ local groups = redis.call('smembers', 'ql:failures')
67
+ for index, group in ipairs(groups) do
68
+ response[group] = redis.call('llen', 'ql:f:' .. group)
69
+ end
70
+ return response
71
+ end
72
+ end
73
+
74
+ function Qless.jobs(now, state, ...)
75
+ assert(state, 'Jobs(): Arg "state" missing')
76
+ if state == 'complete' then
77
+ local offset = assert(tonumber(arg[1] or 0),
78
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[1]))
79
+ local count = assert(tonumber(arg[2] or 25),
80
+ 'Jobs(): Arg "count" not a number: ' .. tostring(arg[2]))
81
+ return redis.call('zrevrange', 'ql:completed', offset,
82
+ offset + count - 1)
83
+ else
84
+ local name = assert(arg[1], 'Jobs(): Arg "queue" missing')
85
+ local offset = assert(tonumber(arg[2] or 0),
86
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[2]))
87
+ local count = assert(tonumber(arg[3] or 25),
88
+ 'Jobs(): Arg "count" not a number: ' .. tostring(arg[3]))
89
+
90
+ local queue = Qless.queue(name)
91
+ if state == 'running' then
92
+ return queue.locks.peek(now, offset, count)
93
+ elseif state == 'stalled' then
94
+ return queue.locks.expired(now, offset, count)
95
+ elseif state == 'scheduled' then
96
+ queue:check_scheduled(now, queue.scheduled.length())
97
+ return queue.scheduled.peek(now, offset, count)
98
+ elseif state == 'depends' then
99
+ return queue.depends.peek(now, offset, count)
100
+ elseif state == 'recurring' then
101
+ return queue.recurring.peek(math.huge, offset, count)
102
+ else
103
+ error('Jobs(): Unknown type "' .. state .. '"')
104
+ end
105
+ end
106
+ end
107
+
108
+ function Qless.track(now, command, jid)
109
+ if command ~= nil then
110
+ assert(jid, 'Track(): Arg "jid" missing')
111
+ assert(Qless.job(jid):exists(), 'Track(): Job does not exist')
112
+ if string.lower(command) == 'track' then
113
+ Qless.publish('track', jid)
114
+ return redis.call('zadd', 'ql:tracked', now, jid)
115
+ elseif string.lower(command) == 'untrack' then
116
+ Qless.publish('untrack', jid)
117
+ return redis.call('zrem', 'ql:tracked', jid)
118
+ else
119
+ error('Track(): Unknown action "' .. command .. '"')
120
+ end
121
+ else
122
+ local response = {
123
+ jobs = {},
124
+ expired = {}
125
+ }
126
+ local jids = redis.call('zrange', 'ql:tracked', 0, -1)
127
+ for index, jid in ipairs(jids) do
128
+ local data = Qless.job(jid):data()
129
+ if data then
130
+ table.insert(response.jobs, data)
131
+ else
132
+ table.insert(response.expired, jid)
133
+ end
134
+ end
135
+ return response
136
+ end
137
+ end
138
+
139
+ function Qless.tag(now, command, ...)
140
+ assert(command,
141
+ 'Tag(): Arg "command" must be "add", "remove", "get" or "top"')
142
+
143
+ if command == 'add' then
144
+ local jid = assert(arg[1], 'Tag(): Arg "jid" missing')
145
+ local tags = redis.call('hget', QlessJob.ns .. jid, 'tags')
146
+ if tags then
147
+ tags = cjson.decode(tags)
148
+ local _tags = {}
149
+ for i,v in ipairs(tags) do _tags[v] = true end
150
+
151
+ for i=2,#arg do
152
+ local tag = arg[i]
153
+ if _tags[tag] == nil then
154
+ _tags[tag] = true
155
+ table.insert(tags, tag)
156
+ end
157
+ redis.call('zadd', 'ql:t:' .. tag, now, jid)
158
+ redis.call('zincrby', 'ql:tags', 1, tag)
159
+ end
160
+
161
+ redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(tags))
162
+ return tags
163
+ else
164
+ error('Tag(): Job ' .. jid .. ' does not exist')
165
+ end
166
+ elseif command == 'remove' then
167
+ local jid = assert(arg[1], 'Tag(): Arg "jid" missing')
168
+ local tags = redis.call('hget', QlessJob.ns .. jid, 'tags')
169
+ if tags then
170
+ tags = cjson.decode(tags)
171
+ local _tags = {}
172
+ for i,v in ipairs(tags) do _tags[v] = true end
173
+
174
+ for i=2,#arg do
175
+ local tag = arg[i]
176
+ _tags[tag] = nil
177
+ redis.call('zrem', 'ql:t:' .. tag, jid)
178
+ redis.call('zincrby', 'ql:tags', -1, tag)
179
+ end
180
+
181
+ local results = {}
182
+ for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
183
+
184
+ redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(results))
185
+ return results
186
+ else
187
+ error('Tag(): Job ' .. jid .. ' does not exist')
188
+ end
189
+ elseif command == 'get' then
190
+ local tag = assert(arg[1], 'Tag(): Arg "tag" missing')
191
+ local offset = assert(tonumber(arg[2] or 0),
192
+ 'Tag(): Arg "offset" not a number: ' .. tostring(arg[2]))
193
+ local count = assert(tonumber(arg[3] or 25),
194
+ 'Tag(): Arg "count" not a number: ' .. tostring(arg[3]))
195
+ return {
196
+ total = redis.call('zcard', 'ql:t:' .. tag),
197
+ jobs = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1)
198
+ }
199
+ elseif command == 'top' then
200
+ local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1]))
201
+ local count = assert(tonumber(arg[2] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(arg[2]))
202
+ return redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count)
203
+ else
204
+ error('Tag(): First argument must be "add", "remove" or "get"')
205
+ end
206
+ end
207
+
208
+ function Qless.cancel(...)
209
+ local dependents = {}
210
+ for _, jid in ipairs(arg) do
211
+ dependents[jid] = redis.call(
212
+ 'smembers', QlessJob.ns .. jid .. '-dependents') or {}
213
+ end
214
+
215
+ for i, jid in ipairs(arg) do
216
+ for j, dep in ipairs(dependents[jid]) do
217
+ if dependents[dep] == nil then
218
+ error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep ..
219
+ ' but is not mentioned to be canceled')
220
+ end
221
+ end
222
+ end
223
+
224
+ for _, jid in ipairs(arg) do
225
+ local state, queue, failure, worker = unpack(redis.call(
226
+ 'hmget', QlessJob.ns .. jid, 'state', 'queue', 'failure', 'worker'))
227
+
228
+ if state ~= 'complete' then
229
+ local encoded = cjson.encode({
230
+ jid = jid,
231
+ worker = worker,
232
+ event = 'canceled',
233
+ queue = queue
234
+ })
235
+ Qless.publish('log', encoded)
236
+
237
+ if worker and (worker ~= '') then
238
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
239
+ Qless.publish('w:' .. worker, encoded)
240
+ end
241
+
242
+ if queue then
243
+ local queue = Qless.queue(queue)
244
+ queue.work.remove(jid)
245
+ queue.locks.remove(jid)
246
+ queue.scheduled.remove(jid)
247
+ queue.depends.remove(jid)
248
+ end
249
+
250
+ for i, j in ipairs(redis.call(
251
+ 'smembers', QlessJob.ns .. jid .. '-dependencies')) do
252
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', jid)
253
+ end
254
+
255
+ redis.call('del', QlessJob.ns .. jid .. '-dependencies')
256
+
257
+ if state == 'failed' then
258
+ failure = cjson.decode(failure)
259
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
260
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
261
+ redis.call('srem', 'ql:failures', failure.group)
262
+ end
263
+ local bin = failure.when - (failure.when % 86400)
264
+ local failed = redis.call(
265
+ 'hget', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed')
266
+ redis.call('hset',
267
+ 'ql:s:stats:' .. bin .. ':' .. queue, 'failed', failed - 1)
268
+ end
269
+
270
+ local tags = cjson.decode(
271
+ redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
272
+ for i, tag in ipairs(tags) do
273
+ redis.call('zrem', 'ql:t:' .. tag, jid)
274
+ redis.call('zincrby', 'ql:tags', -1, tag)
275
+ end
276
+
277
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
278
+ Qless.publish('canceled', jid)
279
+ end
280
+
281
+ redis.call('del', QlessJob.ns .. jid)
282
+ redis.call('del', QlessJob.ns .. jid .. '-history')
283
+ end
284
+ end
285
+
286
+ return arg
287
+ end
288
+
289
+
290
+ Qless.config.defaults = {
291
+ ['application'] = 'qless',
292
+ ['heartbeat'] = 60,
293
+ ['grace-period'] = 10,
294
+ ['stats-history'] = 30,
295
+ ['histogram-history'] = 7,
296
+ ['jobs-history-count'] = 50000,
297
+ ['jobs-history'] = 604800
298
+ }
299
+
300
+ Qless.config.get = function(key, default)
301
+ if key then
302
+ return redis.call('hget', 'ql:config', key) or
303
+ Qless.config.defaults[key] or default
304
+ else
305
+ local reply = redis.call('hgetall', 'ql:config')
306
+ for i = 1, #reply, 2 do
307
+ Qless.config.defaults[reply[i]] = reply[i + 1]
308
+ end
309
+ return Qless.config.defaults
310
+ end
311
+ end
312
+
313
+ Qless.config.set = function(option, value)
314
+ assert(option, 'config.set(): Arg "option" missing')
315
+ assert(value , 'config.set(): Arg "value" missing')
316
+ Qless.publish('log', cjson.encode({
317
+ event = 'config_set',
318
+ option = option,
319
+ value = value
320
+ }))
321
+
322
+ redis.call('hset', 'ql:config', option, value)
323
+ end
324
+
325
+ Qless.config.unset = function(option)
326
+ assert(option, 'config.unset(): Arg "option" missing')
327
+ Qless.publish('log', cjson.encode({
328
+ event = 'config_unset',
329
+ option = option
330
+ }))
331
+
332
+ redis.call('hdel', 'ql:config', option)
333
+ end
334
+
335
+ function QlessJob:data(...)
336
+ local job = redis.call(
337
+ 'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue',
338
+ 'worker', 'priority', 'expires', 'retries', 'remaining', 'data',
339
+ 'tags', 'failure', 'spawned_from_jid')
340
+
341
+ if not job[1] then
342
+ return nil
343
+ end
344
+
345
+ local data = {
346
+ jid = job[1],
347
+ klass = job[2],
348
+ state = job[3],
349
+ queue = job[4],
350
+ worker = job[5] or '',
351
+ tracked = redis.call(
352
+ 'zscore', 'ql:tracked', self.jid) ~= false,
353
+ priority = tonumber(job[6]),
354
+ expires = tonumber(job[7]) or 0,
355
+ retries = tonumber(job[8]),
356
+ remaining = math.floor(tonumber(job[9])),
357
+ data = job[10],
358
+ tags = cjson.decode(job[11]),
359
+ history = self:history(),
360
+ failure = cjson.decode(job[12] or '{}'),
361
+ spawned_from_jid = job[13],
362
+ dependents = redis.call(
363
+ 'smembers', QlessJob.ns .. self.jid .. '-dependents'),
364
+ dependencies = redis.call(
365
+ 'smembers', QlessJob.ns .. self.jid .. '-dependencies')
366
+ }
367
+
368
+ if #arg > 0 then
369
+ local response = {}
370
+ for index, key in ipairs(arg) do
371
+ table.insert(response, data[key])
372
+ end
373
+ return response
374
+ else
375
+ return data
376
+ end
377
+ end
378
+
379
+ function QlessJob:complete(now, worker, queue, data, ...)
380
+ assert(worker, 'Complete(): Arg "worker" missing')
381
+ assert(queue , 'Complete(): Arg "queue" missing')
382
+ data = assert(cjson.decode(data),
383
+ 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data))
384
+
385
+ local options = {}
386
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
387
+
388
+ local nextq = options['next']
389
+ local delay = assert(tonumber(options['delay'] or 0))
390
+ local depends = assert(cjson.decode(options['depends'] or '[]'),
391
+ 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends']))
392
+
393
+ if options['delay'] and nextq == nil then
394
+ error('Complete(): "delay" cannot be used without a "next".')
395
+ end
396
+
397
+ if options['depends'] and nextq == nil then
398
+ error('Complete(): "depends" cannot be used without a "next".')
399
+ end
400
+
401
+ local bin = now - (now % 86400)
402
+
403
+ local lastworker, state, priority, retries, current_queue = unpack(
404
+ redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state',
405
+ 'priority', 'retries', 'queue'))
406
+
407
+ if lastworker == false then
408
+ error('Complete(): Job does not exist')
409
+ elseif (state ~= 'running') then
410
+ error('Complete(): Job is not currently running: ' .. state)
411
+ elseif lastworker ~= worker then
412
+ error('Complete(): Job has been handed out to another worker: ' ..
413
+ tostring(lastworker))
414
+ elseif queue ~= current_queue then
415
+ error('Complete(): Job running in another queue: ' ..
416
+ tostring(current_queue))
417
+ end
418
+
419
+ self:history(now, 'done')
420
+
421
+ if data then
422
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
423
+ end
424
+
425
+ local queue_obj = Qless.queue(queue)
426
+ queue_obj.work.remove(self.jid)
427
+ queue_obj.locks.remove(self.jid)
428
+ queue_obj.scheduled.remove(self.jid)
429
+
430
+ local time = tonumber(
431
+ redis.call('hget', QlessJob.ns .. self.jid, 'time') or now)
432
+ local waiting = now - time
433
+ Qless.queue(queue):stat(now, 'run', waiting)
434
+ redis.call('hset', QlessJob.ns .. self.jid,
435
+ 'time', string.format("%.20f", now))
436
+
437
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
438
+
439
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
440
+ Qless.publish('completed', self.jid)
441
+ end
442
+
443
+ if nextq then
444
+ queue_obj = Qless.queue(nextq)
445
+ Qless.publish('log', cjson.encode({
446
+ jid = self.jid,
447
+ event = 'advanced',
448
+ queue = queue,
449
+ to = nextq
450
+ }))
451
+
452
+ self:history(now, 'put', {q = nextq})
453
+
454
+ if redis.call('zscore', 'ql:queues', nextq) == false then
455
+ redis.call('zadd', 'ql:queues', now, nextq)
456
+ end
457
+
458
+ redis.call('hmset', QlessJob.ns .. self.jid,
459
+ 'state', 'waiting',
460
+ 'worker', '',
461
+ 'failure', '{}',
462
+ 'queue', nextq,
463
+ 'expires', 0,
464
+ 'remaining', tonumber(retries))
465
+
466
+ if (delay > 0) and (#depends == 0) then
467
+ queue_obj.scheduled.add(now + delay, self.jid)
468
+ return 'scheduled'
469
+ else
470
+ local count = 0
471
+ for i, j in ipairs(depends) do
472
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
473
+ if (state and state ~= 'complete') then
474
+ count = count + 1
475
+ redis.call(
476
+ 'sadd', QlessJob.ns .. j .. '-dependents',self.jid)
477
+ redis.call(
478
+ 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j)
479
+ end
480
+ end
481
+ if count > 0 then
482
+ queue_obj.depends.add(now, self.jid)
483
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'depends')
484
+ if delay > 0 then
485
+ queue_obj.depends.add(now, self.jid)
486
+ redis.call('hset', QlessJob.ns .. self.jid, 'scheduled', now + delay)
487
+ end
488
+ return 'depends'
489
+ else
490
+ queue_obj.work.add(now, priority, self.jid)
491
+ return 'waiting'
492
+ end
493
+ end
494
+ else
495
+ Qless.publish('log', cjson.encode({
496
+ jid = self.jid,
497
+ event = 'completed',
498
+ queue = queue
499
+ }))
500
+
501
+ redis.call('hmset', QlessJob.ns .. self.jid,
502
+ 'state', 'complete',
503
+ 'worker', '',
504
+ 'failure', '{}',
505
+ 'queue', '',
506
+ 'expires', 0,
507
+ 'remaining', tonumber(retries))
508
+
509
+ local count = Qless.config.get('jobs-history-count')
510
+ local time = Qless.config.get('jobs-history')
511
+
512
+ count = tonumber(count or 50000)
513
+ time = tonumber(time or 7 * 24 * 60 * 60)
514
+
515
+ redis.call('zadd', 'ql:completed', now, self.jid)
516
+
517
+ local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
518
+ for index, jid in ipairs(jids) do
519
+ local tags = cjson.decode(
520
+ redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
521
+ for i, tag in ipairs(tags) do
522
+ redis.call('zrem', 'ql:t:' .. tag, jid)
523
+ redis.call('zincrby', 'ql:tags', -1, tag)
524
+ end
525
+ redis.call('del', QlessJob.ns .. jid)
526
+ redis.call('del', QlessJob.ns .. jid .. '-history')
527
+ end
528
+ redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
529
+
530
+ jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
531
+ for index, jid in ipairs(jids) do
532
+ local tags = cjson.decode(
533
+ redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
534
+ for i, tag in ipairs(tags) do
535
+ redis.call('zrem', 'ql:t:' .. tag, jid)
536
+ redis.call('zincrby', 'ql:tags', -1, tag)
537
+ end
538
+ redis.call('del', QlessJob.ns .. jid)
539
+ redis.call('del', QlessJob.ns .. jid .. '-history')
540
+ end
541
+ redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
542
+
543
+ for i, j in ipairs(redis.call(
544
+ 'smembers', QlessJob.ns .. self.jid .. '-dependents')) do
545
+ redis.call('srem', QlessJob.ns .. j .. '-dependencies', self.jid)
546
+ if redis.call(
547
+ 'scard', QlessJob.ns .. j .. '-dependencies') == 0 then
548
+ local q, p, scheduled = unpack(
549
+ redis.call('hmget', QlessJob.ns .. j, 'queue', 'priority', 'scheduled'))
550
+ if q then
551
+ local queue = Qless.queue(q)
552
+ queue.depends.remove(j)
553
+ if scheduled then
554
+ queue.scheduled.add(scheduled, j)
555
+ redis.call('hset', QlessJob.ns .. j, 'state', 'scheduled')
556
+ redis.call('hdel', QlessJob.ns .. j, 'scheduled')
557
+ else
558
+ queue.work.add(now, p, j)
559
+ redis.call('hset', QlessJob.ns .. j, 'state', 'waiting')
560
+ end
561
+ end
562
+ end
563
+ end
564
+
565
+ redis.call('del', QlessJob.ns .. self.jid .. '-dependents')
566
+
567
+ return 'complete'
568
+ end
569
+ end
570
+
571
+ function QlessJob:fail(now, worker, group, message, data)
572
+ local worker = assert(worker , 'Fail(): Arg "worker" missing')
573
+ local group = assert(group , 'Fail(): Arg "group" missing')
574
+ local message = assert(message , 'Fail(): Arg "message" missing')
575
+
576
+ local bin = now - (now % 86400)
577
+
578
+ if data then
579
+ data = cjson.decode(data)
580
+ end
581
+
582
+ local queue, state, oldworker = unpack(redis.call(
583
+ 'hmget', QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
584
+
585
+ if not state then
586
+ error('Fail(): Job does not exist')
587
+ elseif state ~= 'running' then
588
+ error('Fail(): Job not currently running: ' .. state)
589
+ elseif worker ~= oldworker then
590
+ error('Fail(): Job running with another worker: ' .. oldworker)
591
+ end
592
+
593
+ Qless.publish('log', cjson.encode({
594
+ jid = self.jid,
595
+ event = 'failed',
596
+ worker = worker,
597
+ group = group,
598
+ message = message
599
+ }))
600
+
601
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
602
+ Qless.publish('failed', self.jid)
603
+ end
604
+
605
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
606
+
607
+ self:history(now, 'failed', {worker = worker, group = group})
608
+
609
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1)
610
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1)
611
+
612
+ local queue_obj = Qless.queue(queue)
613
+ queue_obj.work.remove(self.jid)
614
+ queue_obj.locks.remove(self.jid)
615
+ queue_obj.scheduled.remove(self.jid)
616
+
617
+ if data then
618
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
619
+ end
620
+
621
+ redis.call('hmset', QlessJob.ns .. self.jid,
622
+ 'state', 'failed',
623
+ 'worker', '',
624
+ 'expires', '',
625
+ 'failure', cjson.encode({
626
+ ['group'] = group,
627
+ ['message'] = message,
628
+ ['when'] = math.floor(now),
629
+ ['worker'] = worker
630
+ }))
631
+
632
+ redis.call('sadd', 'ql:failures', group)
633
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
634
+
635
+
636
+ return self.jid
637
+ end
638
+
639
+ function QlessJob:retry(now, queue, worker, delay, group, message)
640
+ assert(queue , 'Retry(): Arg "queue" missing')
641
+ assert(worker, 'Retry(): Arg "worker" missing')
642
+ delay = assert(tonumber(delay or 0),
643
+ 'Retry(): Arg "delay" not a number: ' .. tostring(delay))
644
+
645
+ local oldqueue, state, retries, oldworker, priority, failure = unpack(
646
+ redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state',
647
+ 'retries', 'worker', 'priority', 'failure'))
648
+
649
+ if oldworker == false then
650
+ error('Retry(): Job does not exist')
651
+ elseif state ~= 'running' then
652
+ error('Retry(): Job is not currently running: ' .. state)
653
+ elseif oldworker ~= worker then
654
+ error('Retry(): Job has been given to another worker: ' .. oldworker)
655
+ end
656
+
657
+ local remaining = tonumber(redis.call(
658
+ 'hincrby', QlessJob.ns .. self.jid, 'remaining', -1))
659
+ redis.call('hdel', QlessJob.ns .. self.jid, 'grace')
660
+
661
+ Qless.queue(oldqueue).locks.remove(self.jid)
662
+
663
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
664
+
665
+ if remaining < 0 then
666
+ local group = group or 'failed-retries-' .. queue
667
+ self:history(now, 'failed', {['group'] = group})
668
+
669
+ redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed',
670
+ 'worker', '',
671
+ 'expires', '')
672
+ if group ~= nil and message ~= nil then
673
+ redis.call('hset', QlessJob.ns .. self.jid,
674
+ 'failure', cjson.encode({
675
+ ['group'] = group,
676
+ ['message'] = message,
677
+ ['when'] = math.floor(now),
678
+ ['worker'] = worker
679
+ })
680
+ )
681
+ else
682
+ redis.call('hset', QlessJob.ns .. self.jid,
683
+ 'failure', cjson.encode({
684
+ ['group'] = group,
685
+ ['message'] =
686
+ 'Job exhausted retries in queue "' .. oldqueue .. '"',
687
+ ['when'] = now,
688
+ ['worker'] = unpack(self:data('worker'))
689
+ }))
690
+ end
691
+
692
+ redis.call('sadd', 'ql:failures', group)
693
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
694
+ local bin = now - (now % 86400)
695
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1)
696
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1)
697
+ else
698
+ local queue_obj = Qless.queue(queue)
699
+ if delay > 0 then
700
+ queue_obj.scheduled.add(now + delay, self.jid)
701
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'scheduled')
702
+ else
703
+ queue_obj.work.add(now, priority, self.jid)
704
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting')
705
+ end
706
+
707
+ if group ~= nil and message ~= nil then
708
+ redis.call('hset', QlessJob.ns .. self.jid,
709
+ 'failure', cjson.encode({
710
+ ['group'] = group,
711
+ ['message'] = message,
712
+ ['when'] = math.floor(now),
713
+ ['worker'] = worker
714
+ })
715
+ )
716
+ end
717
+ end
718
+
719
+ return math.floor(remaining)
720
+ end
721
+
722
+ function QlessJob:depends(now, command, ...)
723
+ assert(command, 'Depends(): Arg "command" missing')
724
+ local state = redis.call('hget', QlessJob.ns .. self.jid, 'state')
725
+ if state ~= 'depends' then
726
+ error('Depends(): Job ' .. self.jid ..
727
+ ' not in the depends state: ' .. tostring(state))
728
+ end
729
+
730
+ if command == 'on' then
731
+ for i, j in ipairs(arg) do
732
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
733
+ if (state and state ~= 'complete') then
734
+ redis.call(
735
+ 'sadd', QlessJob.ns .. j .. '-dependents' , self.jid)
736
+ redis.call(
737
+ 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j)
738
+ end
739
+ end
740
+ return true
741
+ elseif command == 'off' then
742
+ if arg[1] == 'all' then
743
+ for i, j in ipairs(redis.call(
744
+ 'smembers', QlessJob.ns .. self.jid .. '-dependencies')) do
745
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid)
746
+ end
747
+ redis.call('del', QlessJob.ns .. self.jid .. '-dependencies')
748
+ local q, p = unpack(redis.call(
749
+ 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority'))
750
+ if q then
751
+ local queue_obj = Qless.queue(q)
752
+ queue_obj.depends.remove(self.jid)
753
+ queue_obj.work.add(now, p, self.jid)
754
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting')
755
+ end
756
+ else
757
+ for i, j in ipairs(arg) do
758
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid)
759
+ redis.call(
760
+ 'srem', QlessJob.ns .. self.jid .. '-dependencies', j)
761
+ if redis.call('scard',
762
+ QlessJob.ns .. self.jid .. '-dependencies') == 0 then
763
+ local q, p = unpack(redis.call(
764
+ 'hmget', QlessJob.ns .. self.jid, 'queue', 'priority'))
765
+ if q then
766
+ local queue_obj = Qless.queue(q)
767
+ queue_obj.depends.remove(self.jid)
768
+ queue_obj.work.add(now, p, self.jid)
769
+ redis.call('hset',
770
+ QlessJob.ns .. self.jid, 'state', 'waiting')
771
+ end
772
+ end
773
+ end
774
+ end
775
+ return true
776
+ else
777
+ error('Depends(): Argument "command" must be "on" or "off"')
778
+ end
779
+ end
780
+
781
+ function QlessJob:heartbeat(now, worker, data)
782
+ assert(worker, 'Heatbeat(): Arg "worker" missing')
783
+
784
+ local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') or ''
785
+ local expires = now + tonumber(
786
+ Qless.config.get(queue .. '-heartbeat') or
787
+ Qless.config.get('heartbeat', 60))
788
+
789
+ if data then
790
+ data = cjson.decode(data)
791
+ end
792
+
793
+ local job_worker, state = unpack(
794
+ redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state'))
795
+ if job_worker == false then
796
+ error('Heartbeat(): Job does not exist')
797
+ elseif state ~= 'running' then
798
+ error('Heartbeat(): Job not currently running: ' .. state)
799
+ elseif job_worker ~= worker or #job_worker == 0 then
800
+ error('Heartbeat(): Job given out to another worker: ' .. job_worker)
801
+ else
802
+ if data then
803
+ redis.call('hmset', QlessJob.ns .. self.jid, 'expires',
804
+ expires, 'worker', worker, 'data', cjson.encode(data))
805
+ else
806
+ redis.call('hmset', QlessJob.ns .. self.jid,
807
+ 'expires', expires, 'worker', worker)
808
+ end
809
+
810
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
811
+
812
+ local queue = Qless.queue(
813
+ redis.call('hget', QlessJob.ns .. self.jid, 'queue'))
814
+ queue.locks.add(expires, self.jid)
815
+ return expires
816
+ end
817
+ end
818
+
819
+ function QlessJob:priority(priority)
820
+ priority = assert(tonumber(priority),
821
+ 'Priority(): Arg "priority" missing or not a number: ' ..
822
+ tostring(priority))
823
+
824
+ local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue')
825
+
826
+ if queue == nil then
827
+ error('Priority(): Job ' .. self.jid .. ' does not exist')
828
+ elseif queue == '' then
829
+ redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority)
830
+ return priority
831
+ else
832
+ local queue_obj = Qless.queue(queue)
833
+ if queue_obj.work.score(self.jid) then
834
+ queue_obj.work.add(0, priority, self.jid)
835
+ end
836
+ redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority)
837
+ return priority
838
+ end
839
+ end
840
+
841
+ function QlessJob:update(data)
842
+ local tmp = {}
843
+ for k, v in pairs(data) do
844
+ table.insert(tmp, k)
845
+ table.insert(tmp, v)
846
+ end
847
+ redis.call('hmset', QlessJob.ns .. self.jid, unpack(tmp))
848
+ end
849
+
850
+ function QlessJob:timeout(now)
851
+ local queue_name, state, worker = unpack(redis.call('hmget',
852
+ QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
853
+ if queue_name == nil then
854
+ error('Timeout(): Job does not exist')
855
+ elseif state ~= 'running' then
856
+ error('Timeout(): Job ' .. self.jid .. ' not running')
857
+ else
858
+ self:history(now, 'timed-out')
859
+ local queue = Qless.queue(queue_name)
860
+ queue.locks.remove(self.jid)
861
+ queue.work.add(now, math.huge, self.jid)
862
+ redis.call('hmset', QlessJob.ns .. self.jid,
863
+ 'state', 'stalled', 'expires', 0)
864
+ local encoded = cjson.encode({
865
+ jid = self.jid,
866
+ event = 'lock_lost',
867
+ worker = worker
868
+ })
869
+ Qless.publish('w:' .. worker, encoded)
870
+ Qless.publish('log', encoded)
871
+ return queue_name
872
+ end
873
+ end
874
+
875
+ function QlessJob:exists()
876
+ return redis.call('exists', QlessJob.ns .. self.jid) == 1
877
+ end
878
+
879
+ function QlessJob:history(now, what, item)
880
+ local history = redis.call('hget', QlessJob.ns .. self.jid, 'history')
881
+ if history then
882
+ history = cjson.decode(history)
883
+ for i, value in ipairs(history) do
884
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
885
+ cjson.encode({math.floor(value.put), 'put', {q = value.q}}))
886
+
887
+ if value.popped then
888
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
889
+ cjson.encode({math.floor(value.popped), 'popped',
890
+ {worker = value.worker}}))
891
+ end
892
+
893
+ if value.failed then
894
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
895
+ cjson.encode(
896
+ {math.floor(value.failed), 'failed', nil}))
897
+ end
898
+
899
+ if value.done then
900
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
901
+ cjson.encode(
902
+ {math.floor(value.done), 'done', nil}))
903
+ end
904
+ end
905
+ redis.call('hdel', QlessJob.ns .. self.jid, 'history')
906
+ end
907
+
908
+ if what == nil then
909
+ local response = {}
910
+ for i, value in ipairs(redis.call('lrange',
911
+ QlessJob.ns .. self.jid .. '-history', 0, -1)) do
912
+ value = cjson.decode(value)
913
+ local dict = value[3] or {}
914
+ dict['when'] = value[1]
915
+ dict['what'] = value[2]
916
+ table.insert(response, dict)
917
+ end
918
+ return response
919
+ else
920
+ local count = tonumber(Qless.config.get('max-job-history', 100))
921
+ if count > 0 then
922
+ local obj = redis.call('lpop', QlessJob.ns .. self.jid .. '-history')
923
+ redis.call('ltrim', QlessJob.ns .. self.jid .. '-history', -count + 2, -1)
924
+ if obj ~= nil then
925
+ redis.call('lpush', QlessJob.ns .. self.jid .. '-history', obj)
926
+ end
927
+ end
928
+ return redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
929
+ cjson.encode({math.floor(now), what, item}))
930
+ end
931
+ end
932
+ function Qless.queue(name)
933
+ assert(name, 'Queue(): no queue name provided')
934
+ local queue = {}
935
+ setmetatable(queue, QlessQueue)
936
+ queue.name = name
937
+
938
+ queue.work = {
939
+ peek = function(count)
940
+ if count == 0 then
941
+ return {}
942
+ end
943
+ local jids = {}
944
+ for index, jid in ipairs(redis.call(
945
+ 'zrevrange', queue:prefix('work'), 0, count - 1)) do
946
+ table.insert(jids, jid)
947
+ end
948
+ return jids
949
+ end, remove = function(...)
950
+ if #arg > 0 then
951
+ return redis.call('zrem', queue:prefix('work'), unpack(arg))
952
+ end
953
+ end, add = function(now, priority, jid)
954
+ return redis.call('zadd',
955
+ queue:prefix('work'), priority - (now / 10000000000), jid)
956
+ end, score = function(jid)
957
+ return redis.call('zscore', queue:prefix('work'), jid)
958
+ end, length = function()
959
+ return redis.call('zcard', queue:prefix('work'))
960
+ end
961
+ }
962
+
963
+ queue.locks = {
964
+ expired = function(now, offset, count)
965
+ return redis.call('zrangebyscore',
966
+ queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count)
967
+ end, peek = function(now, offset, count)
968
+ return redis.call('zrangebyscore', queue:prefix('locks'),
969
+ now, math.huge, 'LIMIT', offset, count)
970
+ end, add = function(expires, jid)
971
+ redis.call('zadd', queue:prefix('locks'), expires, jid)
972
+ end, remove = function(...)
973
+ if #arg > 0 then
974
+ return redis.call('zrem', queue:prefix('locks'), unpack(arg))
975
+ end
976
+ end, running = function(now)
977
+ return redis.call('zcount', queue:prefix('locks'), now, math.huge)
978
+ end, length = function(now)
979
+ if now then
980
+ return redis.call('zcount', queue:prefix('locks'), 0, now)
981
+ else
982
+ return redis.call('zcard', queue:prefix('locks'))
983
+ end
984
+ end
985
+ }
986
+
987
+ queue.depends = {
988
+ peek = function(now, offset, count)
989
+ return redis.call('zrange',
990
+ queue:prefix('depends'), offset, offset + count - 1)
991
+ end, add = function(now, jid)
992
+ redis.call('zadd', queue:prefix('depends'), now, jid)
993
+ end, remove = function(...)
994
+ if #arg > 0 then
995
+ return redis.call('zrem', queue:prefix('depends'), unpack(arg))
996
+ end
997
+ end, length = function()
998
+ return redis.call('zcard', queue:prefix('depends'))
999
+ end
1000
+ }
1001
+
1002
+ queue.scheduled = {
1003
+ peek = function(now, offset, count)
1004
+ return redis.call('zrange',
1005
+ queue:prefix('scheduled'), offset, offset + count - 1)
1006
+ end, ready = function(now, offset, count)
1007
+ return redis.call('zrangebyscore',
1008
+ queue:prefix('scheduled'), 0, now, 'LIMIT', offset, count)
1009
+ end, add = function(when, jid)
1010
+ redis.call('zadd', queue:prefix('scheduled'), when, jid)
1011
+ end, remove = function(...)
1012
+ if #arg > 0 then
1013
+ return redis.call('zrem', queue:prefix('scheduled'), unpack(arg))
1014
+ end
1015
+ end, length = function()
1016
+ return redis.call('zcard', queue:prefix('scheduled'))
1017
+ end
1018
+ }
1019
+
1020
+ queue.recurring = {
1021
+ peek = function(now, offset, count)
1022
+ return redis.call('zrangebyscore', queue:prefix('recur'),
1023
+ 0, now, 'LIMIT', offset, count)
1024
+ end, ready = function(now, offset, count)
1025
+ end, add = function(when, jid)
1026
+ redis.call('zadd', queue:prefix('recur'), when, jid)
1027
+ end, remove = function(...)
1028
+ if #arg > 0 then
1029
+ return redis.call('zrem', queue:prefix('recur'), unpack(arg))
1030
+ end
1031
+ end, update = function(increment, jid)
1032
+ redis.call('zincrby', queue:prefix('recur'), increment, jid)
1033
+ end, score = function(jid)
1034
+ return redis.call('zscore', queue:prefix('recur'), jid)
1035
+ end, length = function()
1036
+ return redis.call('zcard', queue:prefix('recur'))
1037
+ end
1038
+ }
1039
+ return queue
1040
+ end
1041
+
1042
+ function QlessQueue:prefix(group)
1043
+ if group then
1044
+ return QlessQueue.ns..self.name..'-'..group
1045
+ else
1046
+ return QlessQueue.ns..self.name
1047
+ end
1048
+ end
1049
+
1050
+ function QlessQueue:stats(now, date)
1051
+ date = assert(tonumber(date),
1052
+ 'Stats(): Arg "date" missing or not a number: '.. (date or 'nil'))
1053
+
1054
+ local bin = date - (date % 86400)
1055
+
1056
+ local histokeys = {
1057
+ 's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59',
1058
+ 'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59',
1059
+ 'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23',
1060
+ 'd1','d2','d3','d4','d5','d6'
1061
+ }
1062
+
1063
+ local mkstats = function(name, bin, queue)
1064
+ local results = {}
1065
+
1066
+ local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
1067
+ local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
1068
+
1069
+ count = tonumber(count) or 0
1070
+ mean = tonumber(mean) or 0
1071
+ vk = tonumber(vk)
1072
+
1073
+ results.count = count or 0
1074
+ results.mean = mean or 0
1075
+ results.histogram = {}
1076
+
1077
+ if not count then
1078
+ results.std = 0
1079
+ else
1080
+ if count > 1 then
1081
+ results.std = math.sqrt(vk / (count - 1))
1082
+ else
1083
+ results.std = 0
1084
+ end
1085
+ end
1086
+
1087
+ local histogram = redis.call('hmget', key, unpack(histokeys))
1088
+ for i=1,#histokeys do
1089
+ table.insert(results.histogram, tonumber(histogram[i]) or 0)
1090
+ end
1091
+ return results
1092
+ end
1093
+
1094
+ local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 'failed', 'failures'))
1095
+ return {
1096
+ retries = tonumber(retries or 0),
1097
+ failed = tonumber(failed or 0),
1098
+ failures = tonumber(failures or 0),
1099
+ wait = mkstats('wait', bin, self.name),
1100
+ run = mkstats('run' , bin, self.name)
1101
+ }
1102
+ end
1103
+
1104
+ function QlessQueue:peek(now, count)
1105
+ count = assert(tonumber(count),
1106
+ 'Peek(): Arg "count" missing or not a number: ' .. tostring(count))
1107
+
1108
+ local jids = self.locks.expired(now, 0, count)
1109
+
1110
+ self:check_recurring(now, count - #jids)
1111
+
1112
+ self:check_scheduled(now, count - #jids)
1113
+
1114
+ table.extend(jids, self.work.peek(count - #jids))
1115
+
1116
+ return jids
1117
+ end
1118
+
1119
+ function QlessQueue:paused()
1120
+ return redis.call('sismember', 'ql:paused_queues', self.name) == 1
1121
+ end
1122
+
1123
+ function QlessQueue.pause(now, ...)
1124
+ redis.call('sadd', 'ql:paused_queues', unpack(arg))
1125
+ end
1126
+
1127
+ function QlessQueue.unpause(...)
1128
+ redis.call('srem', 'ql:paused_queues', unpack(arg))
1129
+ end
1130
+
1131
+ function QlessQueue:pop(now, worker, count)
1132
+ assert(worker, 'Pop(): Arg "worker" missing')
1133
+ count = assert(tonumber(count),
1134
+ 'Pop(): Arg "count" missing or not a number: ' .. tostring(count))
1135
+
1136
+ local expires = now + tonumber(
1137
+ Qless.config.get(self.name .. '-heartbeat') or
1138
+ Qless.config.get('heartbeat', 60))
1139
+
1140
+ if self:paused() then
1141
+ return {}
1142
+ end
1143
+
1144
+ redis.call('zadd', 'ql:workers', now, worker)
1145
+
1146
+ local max_concurrency = tonumber(
1147
+ Qless.config.get(self.name .. '-max-concurrency', 0))
1148
+
1149
+ if max_concurrency > 0 then
1150
+ local allowed = math.max(0, max_concurrency - self.locks.running(now))
1151
+ count = math.min(allowed, count)
1152
+ if count == 0 then
1153
+ return {}
1154
+ end
1155
+ end
1156
+
1157
+ local jids = self:invalidate_locks(now, count)
1158
+
1159
+ self:check_recurring(now, count - #jids)
1160
+
1161
+ self:check_scheduled(now, count - #jids)
1162
+
1163
+ table.extend(jids, self.work.peek(count - #jids))
1164
+
1165
+ local state
1166
+ for index, jid in ipairs(jids) do
1167
+ local job = Qless.job(jid)
1168
+ state = unpack(job:data('state'))
1169
+ job:history(now, 'popped', {worker = worker})
1170
+
1171
+ local time = tonumber(
1172
+ redis.call('hget', QlessJob.ns .. jid, 'time') or now)
1173
+ local waiting = now - time
1174
+ self:stat(now, 'wait', waiting)
1175
+ redis.call('hset', QlessJob.ns .. jid,
1176
+ 'time', string.format("%.20f", now))
1177
+
1178
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
1179
+
1180
+ job:update({
1181
+ worker = worker,
1182
+ expires = expires,
1183
+ state = 'running'
1184
+ })
1185
+
1186
+ self.locks.add(expires, jid)
1187
+
1188
+ local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
1189
+ if tracked then
1190
+ Qless.publish('popped', jid)
1191
+ end
1192
+ end
1193
+
1194
+ self.work.remove(unpack(jids))
1195
+
1196
+ return jids
1197
+ end
1198
+
1199
+ function QlessQueue:stat(now, stat, val)
1200
+ local bin = now - (now % 86400)
1201
+ local key = 'ql:s:' .. stat .. ':' .. bin .. ':' .. self.name
1202
+
1203
+ local count, mean, vk = unpack(
1204
+ redis.call('hmget', key, 'total', 'mean', 'vk'))
1205
+
1206
+ count = count or 0
1207
+ if count == 0 then
1208
+ mean = val
1209
+ vk = 0
1210
+ count = 1
1211
+ else
1212
+ count = count + 1
1213
+ local oldmean = mean
1214
+ mean = mean + (val - mean) / count
1215
+ vk = vk + (val - mean) * (val - oldmean)
1216
+ end
1217
+
1218
+ val = math.floor(val)
1219
+ if val < 60 then -- seconds
1220
+ redis.call('hincrby', key, 's' .. val, 1)
1221
+ elseif val < 3600 then -- minutes
1222
+ redis.call('hincrby', key, 'm' .. math.floor(val / 60), 1)
1223
+ elseif val < 86400 then -- hours
1224
+ redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
1225
+ else -- days
1226
+ redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
1227
+ end
1228
+ redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
1229
+ end
1230
+
1231
+ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
1232
+ assert(jid , 'Put(): Arg "jid" missing')
1233
+ assert(klass, 'Put(): Arg "klass" missing')
1234
+ local data = assert(cjson.decode(raw_data),
1235
+ 'Put(): Arg "data" missing or not JSON: ' .. tostring(raw_data))
1236
+ delay = assert(tonumber(delay),
1237
+ 'Put(): Arg "delay" not a number: ' .. tostring(delay))
1238
+
1239
+ if #arg % 2 == 1 then
1240
+ error('Odd number of additional args: ' .. tostring(arg))
1241
+ end
1242
+ local options = {}
1243
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
1244
+
1245
+ local job = Qless.job(jid)
1246
+ local priority, tags, oldqueue, state, failure, retries, oldworker =
1247
+ unpack(redis.call('hmget', QlessJob.ns .. jid, 'priority', 'tags',
1248
+ 'queue', 'state', 'failure', 'retries', 'worker'))
1249
+
1250
+ if tags then
1251
+ Qless.tag(now, 'remove', jid, unpack(cjson.decode(tags)))
1252
+ end
1253
+
1254
+ retries = assert(tonumber(options['retries'] or retries or 5) ,
1255
+ 'Put(): Arg "retries" not a number: ' .. tostring(options['retries']))
1256
+ tags = assert(cjson.decode(options['tags'] or tags or '[]' ),
1257
+ 'Put(): Arg "tags" not JSON' .. tostring(options['tags']))
1258
+ priority = assert(tonumber(options['priority'] or priority or 0),
1259
+ 'Put(): Arg "priority" not a number' .. tostring(options['priority']))
1260
+ local depends = assert(cjson.decode(options['depends'] or '[]') ,
1261
+ 'Put(): Arg "depends" not JSON: ' .. tostring(options['depends']))
1262
+
1263
+ if #depends > 0 then
1264
+ local new = {}
1265
+ for _, d in ipairs(depends) do new[d] = 1 end
1266
+
1267
+ local original = redis.call(
1268
+ 'smembers', QlessJob.ns .. jid .. '-dependencies')
1269
+ for _, dep in pairs(original) do
1270
+ if new[dep] == nil then
1271
+ redis.call('srem', QlessJob.ns .. dep .. '-dependents' , jid)
1272
+ redis.call('srem', QlessJob.ns .. jid .. '-dependencies', dep)
1273
+ end
1274
+ end
1275
+ end
1276
+
1277
+ Qless.publish('log', cjson.encode({
1278
+ jid = jid,
1279
+ event = 'put',
1280
+ queue = self.name
1281
+ }))
1282
+
1283
+ job:history(now, 'put', {q = self.name})
1284
+
1285
+ if oldqueue then
1286
+ local queue_obj = Qless.queue(oldqueue)
1287
+ queue_obj.work.remove(jid)
1288
+ queue_obj.locks.remove(jid)
1289
+ queue_obj.depends.remove(jid)
1290
+ queue_obj.scheduled.remove(jid)
1291
+ end
1292
+
1293
+ if oldworker and oldworker ~= '' then
1294
+ redis.call('zrem', 'ql:w:' .. oldworker .. ':jobs', jid)
1295
+ if oldworker ~= worker then
1296
+ local encoded = cjson.encode({
1297
+ jid = jid,
1298
+ event = 'lock_lost',
1299
+ worker = oldworker
1300
+ })
1301
+ Qless.publish('w:' .. oldworker, encoded)
1302
+ Qless.publish('log', encoded)
1303
+ end
1304
+ end
1305
+
1306
+ if state == 'complete' then
1307
+ redis.call('zrem', 'ql:completed', jid)
1308
+ end
1309
+
1310
+ for i, tag in ipairs(tags) do
1311
+ redis.call('zadd', 'ql:t:' .. tag, now, jid)
1312
+ redis.call('zincrby', 'ql:tags', 1, tag)
1313
+ end
1314
+
1315
+ if state == 'failed' then
1316
+ failure = cjson.decode(failure)
1317
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
1318
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
1319
+ redis.call('srem', 'ql:failures', failure.group)
1320
+ end
1321
+ local bin = failure.when - (failure.when % 86400)
1322
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , -1)
1323
+ end
1324
+
1325
+ redis.call('hmset', QlessJob.ns .. jid,
1326
+ 'jid' , jid,
1327
+ 'klass' , klass,
1328
+ 'data' , raw_data,
1329
+ 'priority' , priority,
1330
+ 'tags' , cjson.encode(tags),
1331
+ 'state' , ((delay > 0) and 'scheduled') or 'waiting',
1332
+ 'worker' , '',
1333
+ 'expires' , 0,
1334
+ 'queue' , self.name,
1335
+ 'retries' , retries,
1336
+ 'remaining', retries,
1337
+ 'time' , string.format("%.20f", now))
1338
+
1339
+ for i, j in ipairs(depends) do
1340
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
1341
+ if (state and state ~= 'complete') then
1342
+ redis.call('sadd', QlessJob.ns .. j .. '-dependents' , jid)
1343
+ redis.call('sadd', QlessJob.ns .. jid .. '-dependencies', j)
1344
+ end
1345
+ end
1346
+
1347
+ if delay > 0 then
1348
+ if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
1349
+ self.depends.add(now, jid)
1350
+ redis.call('hmset', QlessJob.ns .. jid,
1351
+ 'state', 'depends',
1352
+ 'scheduled', now + delay)
1353
+ else
1354
+ self.scheduled.add(now + delay, jid)
1355
+ end
1356
+ else
1357
+ if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
1358
+ self.depends.add(now, jid)
1359
+ redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
1360
+ else
1361
+ self.work.add(now, priority, jid)
1362
+ end
1363
+ end
1364
+
1365
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1366
+ redis.call('zadd', 'ql:queues', now, self.name)
1367
+ end
1368
+
1369
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1370
+ Qless.publish('put', jid)
1371
+ end
1372
+
1373
+ return jid
1374
+ end
1375
+
1376
+ function QlessQueue:unfail(now, group, count)
1377
+ assert(group, 'Unfail(): Arg "group" missing')
1378
+ count = assert(tonumber(count or 25),
1379
+ 'Unfail(): Arg "count" not a number: ' .. tostring(count))
1380
+
1381
+ local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1)
1382
+
1383
+ local toinsert = {}
1384
+ for index, jid in ipairs(jids) do
1385
+ local job = Qless.job(jid)
1386
+ local data = job:data()
1387
+ job:history(now, 'put', {q = self.name})
1388
+ redis.call('hmset', QlessJob.ns .. data.jid,
1389
+ 'state' , 'waiting',
1390
+ 'worker' , '',
1391
+ 'expires' , 0,
1392
+ 'queue' , self.name,
1393
+ 'remaining', data.retries or 5)
1394
+ self.work.add(now, data.priority, data.jid)
1395
+ end
1396
+
1397
+ redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1)
1398
+ if (redis.call('llen', 'ql:f:' .. group) == 0) then
1399
+ redis.call('srem', 'ql:failures', group)
1400
+ end
1401
+
1402
+ return #jids
1403
+ end
1404
+
1405
+ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1406
+ assert(jid , 'RecurringJob On(): Arg "jid" missing')
1407
+ assert(klass, 'RecurringJob On(): Arg "klass" missing')
1408
+ assert(spec , 'RecurringJob On(): Arg "spec" missing')
1409
+ local data = assert(cjson.decode(raw_data),
1410
+ 'RecurringJob On(): Arg "data" not JSON: ' .. tostring(raw_data))
1411
+
1412
+ if spec == 'interval' then
1413
+ local interval = assert(tonumber(arg[1]),
1414
+ 'Recur(): Arg "interval" not a number: ' .. tostring(arg[1]))
1415
+ local offset = assert(tonumber(arg[2]),
1416
+ 'Recur(): Arg "offset" not a number: ' .. tostring(arg[2]))
1417
+ if interval <= 0 then
1418
+ error('Recur(): Arg "interval" must be greater than 0')
1419
+ end
1420
+
1421
+ if #arg % 2 == 1 then
1422
+ error('Odd number of additional args: ' .. tostring(arg))
1423
+ end
1424
+
1425
+ local options = {}
1426
+ for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end
1427
+ options.tags = assert(cjson.decode(options.tags or '{}'),
1428
+ 'Recur(): Arg "tags" must be JSON string array: ' .. tostring(
1429
+ options.tags))
1430
+ options.priority = assert(tonumber(options.priority or 0),
1431
+ 'Recur(): Arg "priority" not a number: ' .. tostring(
1432
+ options.priority))
1433
+ options.retries = assert(tonumber(options.retries or 0),
1434
+ 'Recur(): Arg "retries" not a number: ' .. tostring(
1435
+ options.retries))
1436
+ options.backlog = assert(tonumber(options.backlog or 0),
1437
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(
1438
+ options.backlog))
1439
+
1440
+ local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
1441
+ count = count or 0
1442
+
1443
+ if old_queue then
1444
+ Qless.queue(old_queue).recurring.remove(jid)
1445
+ end
1446
+
1447
+ redis.call('hmset', 'ql:r:' .. jid,
1448
+ 'jid' , jid,
1449
+ 'klass' , klass,
1450
+ 'data' , raw_data,
1451
+ 'priority', options.priority,
1452
+ 'tags' , cjson.encode(options.tags or {}),
1453
+ 'state' , 'recur',
1454
+ 'queue' , self.name,
1455
+ 'type' , 'interval',
1456
+ 'count' , count,
1457
+ 'interval', interval,
1458
+ 'retries' , options.retries,
1459
+ 'backlog' , options.backlog)
1460
+ self.recurring.add(now + offset, jid)
1461
+
1462
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1463
+ redis.call('zadd', 'ql:queues', now, self.name)
1464
+ end
1465
+
1466
+ return jid
1467
+ else
1468
+ error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
1469
+ end
1470
+ end
1471
+
1472
+ function QlessQueue:length()
1473
+ return self.locks.length() + self.work.length() + self.scheduled.length()
1474
+ end
1475
+
1476
+ function QlessQueue:check_recurring(now, count)
1477
+ local moved = 0
1478
+ local r = self.recurring.peek(now, 0, count)
1479
+ for index, jid in ipairs(r) do
1480
+ local klass, data, priority, tags, retries, interval, backlog = unpack(
1481
+ redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
1482
+ 'tags', 'retries', 'interval', 'backlog'))
1483
+ local _tags = cjson.decode(tags)
1484
+ local score = math.floor(tonumber(self.recurring.score(jid)))
1485
+ interval = tonumber(interval)
1486
+
1487
+ backlog = tonumber(backlog or 0)
1488
+ if backlog ~= 0 then
1489
+ local num = ((now - score) / interval)
1490
+ if num > backlog then
1491
+ score = score + (
1492
+ math.ceil(num - backlog) * interval
1493
+ )
1494
+ end
1495
+ end
1496
+
1497
+ while (score <= now) and (moved < count) do
1498
+ local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
1499
+ moved = moved + 1
1500
+
1501
+ local child_jid = jid .. '-' .. count
1502
+
1503
+ for i, tag in ipairs(_tags) do
1504
+ redis.call('zadd', 'ql:t:' .. tag, now, child_jid)
1505
+ redis.call('zincrby', 'ql:tags', 1, tag)
1506
+ end
1507
+
1508
+ redis.call('hmset', QlessJob.ns .. child_jid,
1509
+ 'jid' , child_jid,
1510
+ 'klass' , klass,
1511
+ 'data' , data,
1512
+ 'priority' , priority,
1513
+ 'tags' , tags,
1514
+ 'state' , 'waiting',
1515
+ 'worker' , '',
1516
+ 'expires' , 0,
1517
+ 'queue' , self.name,
1518
+ 'retries' , retries,
1519
+ 'remaining' , retries,
1520
+ 'time' , string.format("%.20f", score),
1521
+ 'spawned_from_jid', jid)
1522
+ Qless.job(child_jid):history(score, 'put', {q = self.name})
1523
+
1524
+ self.work.add(score, priority, child_jid)
1525
+
1526
+ score = score + interval
1527
+ self.recurring.add(score, jid)
1528
+ end
1529
+ end
1530
+ end
1531
+
1532
+ function QlessQueue:check_scheduled(now, count)
1533
+ local scheduled = self.scheduled.ready(now, 0, count)
1534
+ for index, jid in ipairs(scheduled) do
1535
+ local priority = tonumber(
1536
+ redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
1537
+ self.work.add(now, priority, jid)
1538
+ self.scheduled.remove(jid)
1539
+
1540
+ redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
1541
+ end
1542
+ end
1543
+
1544
+ function QlessQueue:invalidate_locks(now, count)
1545
+ local jids = {}
1546
+ for index, jid in ipairs(self.locks.expired(now, 0, count)) do
1547
+ local worker, failure = unpack(
1548
+ redis.call('hmget', QlessJob.ns .. jid, 'worker', 'failure'))
1549
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
1550
+
1551
+ local grace_period = tonumber(Qless.config.get('grace-period'))
1552
+
1553
+ local courtesy_sent = tonumber(
1554
+ redis.call('hget', QlessJob.ns .. jid, 'grace') or 0)
1555
+
1556
+ local send_message = (courtesy_sent ~= 1)
1557
+ local invalidate = not send_message
1558
+
1559
+ if grace_period <= 0 then
1560
+ send_message = true
1561
+ invalidate = true
1562
+ end
1563
+
1564
+ if send_message then
1565
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1566
+ Qless.publish('stalled', jid)
1567
+ end
1568
+ Qless.job(jid):history(now, 'timed-out')
1569
+ redis.call('hset', QlessJob.ns .. jid, 'grace', 1)
1570
+
1571
+ local encoded = cjson.encode({
1572
+ jid = jid,
1573
+ event = 'lock_lost',
1574
+ worker = worker
1575
+ })
1576
+ Qless.publish('w:' .. worker, encoded)
1577
+ Qless.publish('log', encoded)
1578
+ self.locks.add(now + grace_period, jid)
1579
+
1580
+ local bin = now - (now % 86400)
1581
+ redis.call('hincrby',
1582
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 1)
1583
+ end
1584
+
1585
+ if invalidate then
1586
+ redis.call('hdel', QlessJob.ns .. jid, 'grace', 0)
1587
+
1588
+ local remaining = tonumber(redis.call(
1589
+ 'hincrby', QlessJob.ns .. jid, 'remaining', -1))
1590
+
1591
+ if remaining < 0 then
1592
+ self.work.remove(jid)
1593
+ self.locks.remove(jid)
1594
+ self.scheduled.remove(jid)
1595
+
1596
+ local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
1597
+ local job = Qless.job(jid)
1598
+ job:history(now, 'failed', {group = group})
1599
+ redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed',
1600
+ 'worker', '',
1601
+ 'expires', '')
1602
+ redis.call('hset', QlessJob.ns .. jid,
1603
+ 'failure', cjson.encode({
1604
+ ['group'] = group,
1605
+ ['message'] =
1606
+ 'Job exhausted retries in queue "' .. self.name .. '"',
1607
+ ['when'] = now,
1608
+ ['worker'] = unpack(job:data('worker'))
1609
+ }))
1610
+
1611
+ redis.call('sadd', 'ql:failures', group)
1612
+ redis.call('lpush', 'ql:f:' .. group, jid)
1613
+
1614
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1615
+ Qless.publish('failed', jid)
1616
+ end
1617
+ Qless.publish('log', cjson.encode({
1618
+ jid = jid,
1619
+ event = 'failed',
1620
+ group = group,
1621
+ worker = worker,
1622
+ message =
1623
+ 'Job exhausted retries in queue "' .. self.name .. '"'
1624
+ }))
1625
+
1626
+ local bin = now - (now % 86400)
1627
+ redis.call('hincrby',
1628
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'failures', 1)
1629
+ redis.call('hincrby',
1630
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , 1)
1631
+ else
1632
+ table.insert(jids, jid)
1633
+ end
1634
+ end
1635
+ end
1636
+
1637
+ return jids
1638
+ end
1639
+
1640
+ function QlessQueue.deregister(...)
1641
+ redis.call('zrem', Qless.ns .. 'queues', unpack(arg))
1642
+ end
1643
+
1644
+ function QlessQueue.counts(now, name)
1645
+ if name then
1646
+ local queue = Qless.queue(name)
1647
+ local stalled = queue.locks.length(now)
1648
+ queue:check_scheduled(now, queue.scheduled.length())
1649
+ return {
1650
+ name = name,
1651
+ waiting = queue.work.length(),
1652
+ stalled = stalled,
1653
+ running = queue.locks.length() - stalled,
1654
+ scheduled = queue.scheduled.length(),
1655
+ depends = queue.depends.length(),
1656
+ recurring = queue.recurring.length(),
1657
+ paused = queue:paused()
1658
+ }
1659
+ else
1660
+ local queues = redis.call('zrange', 'ql:queues', 0, -1)
1661
+ local response = {}
1662
+ for index, qname in ipairs(queues) do
1663
+ table.insert(response, QlessQueue.counts(now, qname))
1664
+ end
1665
+ return response
1666
+ end
1667
+ end
1668
+ function QlessRecurringJob:data()
1669
+ local job = redis.call(
1670
+ 'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue',
1671
+ 'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog')
1672
+
1673
+ if not job[1] then
1674
+ return nil
1675
+ end
1676
+
1677
+ return {
1678
+ jid = job[1],
1679
+ klass = job[2],
1680
+ state = job[3],
1681
+ queue = job[4],
1682
+ priority = tonumber(job[5]),
1683
+ interval = tonumber(job[6]),
1684
+ retries = tonumber(job[7]),
1685
+ count = tonumber(job[8]),
1686
+ data = job[9],
1687
+ tags = cjson.decode(job[10]),
1688
+ backlog = tonumber(job[11] or 0)
1689
+ }
1690
+ end
1691
+
1692
+ function QlessRecurringJob:update(now, ...)
1693
+ local options = {}
1694
+ if redis.call('exists', 'ql:r:' .. self.jid) ~= 0 then
1695
+ for i = 1, #arg, 2 do
1696
+ local key = arg[i]
1697
+ local value = arg[i+1]
1698
+ assert(value, 'No value provided for ' .. tostring(key))
1699
+ if key == 'priority' or key == 'interval' or key == 'retries' then
1700
+ value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value))
1701
+ if key == 'interval' then
1702
+ local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. self.jid, 'queue', 'interval'))
1703
+ Qless.queue(queue).recurring.update(
1704
+ value - tonumber(interval), self.jid)
1705
+ end
1706
+ redis.call('hset', 'ql:r:' .. self.jid, key, value)
1707
+ elseif key == 'data' then
1708
+ assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value))
1709
+ redis.call('hset', 'ql:r:' .. self.jid, 'data', value)
1710
+ elseif key == 'klass' then
1711
+ redis.call('hset', 'ql:r:' .. self.jid, 'klass', value)
1712
+ elseif key == 'queue' then
1713
+ local queue_obj = Qless.queue(
1714
+ redis.call('hget', 'ql:r:' .. self.jid, 'queue'))
1715
+ local score = queue_obj.recurring.score(self.jid)
1716
+ queue_obj.recurring.remove(self.jid)
1717
+ Qless.queue(value).recurring.add(score, self.jid)
1718
+ redis.call('hset', 'ql:r:' .. self.jid, 'queue', value)
1719
+ if redis.call('zscore', 'ql:queues', value) == false then
1720
+ redis.call('zadd', 'ql:queues', now, value)
1721
+ end
1722
+ elseif key == 'backlog' then
1723
+ value = assert(tonumber(value),
1724
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(value))
1725
+ redis.call('hset', 'ql:r:' .. self.jid, 'backlog', value)
1726
+ else
1727
+ error('Recur(): Unrecognized option "' .. key .. '"')
1728
+ end
1729
+ end
1730
+ return true
1731
+ else
1732
+ error('Recur(): No recurring job ' .. self.jid)
1733
+ end
1734
+ end
1735
+
1736
+ function QlessRecurringJob:tag(...)
1737
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
1738
+ if tags then
1739
+ tags = cjson.decode(tags)
1740
+ local _tags = {}
1741
+ for i,v in ipairs(tags) do _tags[v] = true end
1742
+
1743
+ for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end
1744
+
1745
+ tags = cjson.encode(tags)
1746
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
1747
+ return tags
1748
+ else
1749
+ error('Tag(): Job ' .. self.jid .. ' does not exist')
1750
+ end
1751
+ end
1752
+
1753
+ function QlessRecurringJob:untag(...)
1754
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
1755
+ if tags then
1756
+ tags = cjson.decode(tags)
1757
+ local _tags = {}
1758
+ for i,v in ipairs(tags) do _tags[v] = true end
1759
+ for i = 1,#arg do _tags[arg[i]] = nil end
1760
+ local results = {}
1761
+ for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
1762
+ tags = cjson.encode(results)
1763
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
1764
+ return tags
1765
+ else
1766
+ error('Untag(): Job ' .. self.jid .. ' does not exist')
1767
+ end
1768
+ end
1769
+
1770
+ function QlessRecurringJob:unrecur()
1771
+ local queue = redis.call('hget', 'ql:r:' .. self.jid, 'queue')
1772
+ if queue then
1773
+ Qless.queue(queue).recurring.remove(self.jid)
1774
+ redis.call('del', 'ql:r:' .. self.jid)
1775
+ return true
1776
+ else
1777
+ return true
1778
+ end
1779
+ end
1780
+ function QlessWorker.deregister(...)
1781
+ redis.call('zrem', 'ql:workers', unpack(arg))
1782
+ end
1783
+
1784
+ function QlessWorker.counts(now, worker)
1785
+ local interval = tonumber(Qless.config.get('max-worker-age', 86400))
1786
+
1787
+ local workers = redis.call('zrangebyscore', 'ql:workers', 0, now - interval)
1788
+ for index, worker in ipairs(workers) do
1789
+ redis.call('del', 'ql:w:' .. worker .. ':jobs')
1790
+ end
1791
+
1792
+ redis.call('zremrangebyscore', 'ql:workers', 0, now - interval)
1793
+
1794
+ if worker then
1795
+ return {
1796
+ jobs = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now),
1797
+ stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0)
1798
+ }
1799
+ else
1800
+ local response = {}
1801
+ local workers = redis.call('zrevrange', 'ql:workers', 0, -1)
1802
+ for index, worker in ipairs(workers) do
1803
+ table.insert(response, {
1804
+ name = worker,
1805
+ jobs = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000),
1806
+ stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now)
1807
+ })
1808
+ end
1809
+ return response
1810
+ end
1811
+ end
1812
+ local QlessAPI = {}
1813
+
1814
+ function QlessAPI.get(now, jid)
1815
+ local data = Qless.job(jid):data()
1816
+ if not data then
1817
+ return nil
1818
+ end
1819
+ return cjson.encode(data)
1820
+ end
1821
+
1822
+ function QlessAPI.multiget(now, ...)
1823
+ local results = {}
1824
+ for i, jid in ipairs(arg) do
1825
+ table.insert(results, Qless.job(jid):data())
1826
+ end
1827
+ return cjson.encode(results)
1828
+ end
1829
+
1830
+ QlessAPI['config.get'] = function(now, key)
1831
+ if not key then
1832
+ return cjson.encode(Qless.config.get(key))
1833
+ else
1834
+ return Qless.config.get(key)
1835
+ end
1836
+ end
1837
+
1838
+ QlessAPI['config.set'] = function(now, key, value)
1839
+ return Qless.config.set(key, value)
1840
+ end
1841
+
1842
+ QlessAPI['config.unset'] = function(now, key)
1843
+ return Qless.config.unset(key)
1844
+ end
1845
+
1846
+ QlessAPI.queues = function(now, queue)
1847
+ return cjson.encode(QlessQueue.counts(now, queue))
1848
+ end
1849
+
1850
+ QlessAPI.complete = function(now, jid, worker, queue, data, ...)
1851
+ return Qless.job(jid):complete(now, worker, queue, data, unpack(arg))
1852
+ end
1853
+
1854
+ QlessAPI.failed = function(now, group, start, limit)
1855
+ return cjson.encode(Qless.failed(group, start, limit))
1856
+ end
1857
+
1858
+ QlessAPI.fail = function(now, jid, worker, group, message, data)
1859
+ return Qless.job(jid):fail(now, worker, group, message, data)
1860
+ end
1861
+
1862
+ QlessAPI.jobs = function(now, state, ...)
1863
+ return Qless.jobs(now, state, unpack(arg))
1864
+ end
1865
+
1866
+ QlessAPI.retry = function(now, jid, queue, worker, delay, group, message)
1867
+ return Qless.job(jid):retry(now, queue, worker, delay, group, message)
1868
+ end
1869
+
1870
+ QlessAPI.depends = function(now, jid, command, ...)
1871
+ return Qless.job(jid):depends(now, command, unpack(arg))
1872
+ end
1873
+
1874
+ QlessAPI.heartbeat = function(now, jid, worker, data)
1875
+ return Qless.job(jid):heartbeat(now, worker, data)
1876
+ end
1877
+
1878
+ QlessAPI.workers = function(now, worker)
1879
+ return cjson.encode(QlessWorker.counts(now, worker))
1880
+ end
1881
+
1882
+ QlessAPI.track = function(now, command, jid)
1883
+ return cjson.encode(Qless.track(now, command, jid))
1884
+ end
1885
+
1886
+ QlessAPI.tag = function(now, command, ...)
1887
+ return cjson.encode(Qless.tag(now, command, unpack(arg)))
1888
+ end
1889
+
1890
+ QlessAPI.stats = function(now, queue, date)
1891
+ return cjson.encode(Qless.queue(queue):stats(now, date))
1892
+ end
1893
+
1894
+ QlessAPI.priority = function(now, jid, priority)
1895
+ return Qless.job(jid):priority(priority)
1896
+ end
1897
+
1898
+ QlessAPI.log = function(now, jid, message, data)
1899
+ assert(jid, "Log(): Argument 'jid' missing")
1900
+ assert(message, "Log(): Argument 'message' missing")
1901
+ if data then
1902
+ data = assert(cjson.decode(data),
1903
+ "Log(): Argument 'data' not cjson: " .. tostring(data))
1904
+ end
1905
+
1906
+ local job = Qless.job(jid)
1907
+ assert(job:exists(), 'Log(): Job ' .. jid .. ' does not exist')
1908
+ job:history(now, message, data)
1909
+ end
1910
+
1911
+ QlessAPI.peek = function(now, queue, count)
1912
+ local jids = Qless.queue(queue):peek(now, count)
1913
+ local response = {}
1914
+ for i, jid in ipairs(jids) do
1915
+ table.insert(response, Qless.job(jid):data())
1916
+ end
1917
+ return cjson.encode(response)
1918
+ end
1919
+
1920
+ QlessAPI.pop = function(now, queue, worker, count)
1921
+ local jids = Qless.queue(queue):pop(now, worker, count)
1922
+ local response = {}
1923
+ for i, jid in ipairs(jids) do
1924
+ table.insert(response, Qless.job(jid):data())
1925
+ end
1926
+ return cjson.encode(response)
1927
+ end
1928
+
1929
+ QlessAPI.pause = function(now, ...)
1930
+ return QlessQueue.pause(now, unpack(arg))
1931
+ end
1932
+
1933
+ QlessAPI.unpause = function(now, ...)
1934
+ return QlessQueue.unpause(unpack(arg))
1935
+ end
1936
+
1937
+ QlessAPI.cancel = function(now, ...)
1938
+ return Qless.cancel(unpack(arg))
1939
+ end
1940
+
1941
+ QlessAPI.timeout = function(now, ...)
1942
+ for _, jid in ipairs(arg) do
1943
+ Qless.job(jid):timeout(now)
1944
+ end
1945
+ end
1946
+
1947
+ QlessAPI.put = function(now, me, queue, jid, klass, data, delay, ...)
1948
+ return Qless.queue(queue):put(now, me, jid, klass, data, delay, unpack(arg))
1949
+ end
1950
+
1951
+ QlessAPI.requeue = function(now, me, queue, jid, ...)
1952
+ local job = Qless.job(jid)
1953
+ assert(job:exists(), 'Requeue(): Job ' .. jid .. ' does not exist')
1954
+ return QlessAPI.put(now, me, queue, jid, unpack(arg))
1955
+ end
1956
+
1957
+ QlessAPI.unfail = function(now, queue, group, count)
1958
+ return Qless.queue(queue):unfail(now, group, count)
1959
+ end
1960
+
1961
+ QlessAPI.recur = function(now, queue, jid, klass, data, spec, ...)
1962
+ return Qless.queue(queue):recur(now, jid, klass, data, spec, unpack(arg))
1963
+ end
1964
+
1965
+ QlessAPI.unrecur = function(now, jid)
1966
+ return Qless.recurring(jid):unrecur()
1967
+ end
1968
+
1969
+ QlessAPI['recur.get'] = function(now, jid)
1970
+ local data = Qless.recurring(jid):data()
1971
+ if not data then
1972
+ return nil
1973
+ end
1974
+ return cjson.encode(data)
1975
+ end
1976
+
1977
+ QlessAPI['recur.update'] = function(now, jid, ...)
1978
+ return Qless.recurring(jid):update(now, unpack(arg))
1979
+ end
1980
+
1981
+ QlessAPI['recur.tag'] = function(now, jid, ...)
1982
+ return Qless.recurring(jid):tag(unpack(arg))
1983
+ end
1984
+
1985
+ QlessAPI['recur.untag'] = function(now, jid, ...)
1986
+ return Qless.recurring(jid):untag(unpack(arg))
1987
+ end
1988
+
1989
+ QlessAPI.length = function(now, queue)
1990
+ return Qless.queue(queue):length()
1991
+ end
1992
+
1993
+ QlessAPI['worker.deregister'] = function(now, ...)
1994
+ return QlessWorker.deregister(unpack(arg))
1995
+ end
1996
+
1997
+ QlessAPI['queue.forget'] = function(now, ...)
1998
+ QlessQueue.deregister(unpack(arg))
1999
+ end
2000
+
2001
+
2002
+ if #KEYS > 0 then error('No Keys should be provided') end
2003
+
2004
+ local command_name = assert(table.remove(ARGV, 1), 'Must provide a command')
2005
+ local command = assert(
2006
+ QlessAPI[command_name], 'Unknown command ' .. command_name)
2007
+
2008
+ local now = tonumber(table.remove(ARGV, 1))
2009
+ local now = assert(
2010
+ now, 'Arg "now" missing or not a number: ' .. (now or 'nil'))
2011
+
2012
+ return command(now, unpack(ARGV))