qless 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/Gemfile +2 -0
  2. data/README.md +42 -3
  3. data/Rakefile +26 -2
  4. data/{bin → exe}/qless-web +3 -2
  5. data/lib/qless.rb +55 -28
  6. data/lib/qless/config.rb +1 -3
  7. data/lib/qless/job.rb +127 -22
  8. data/lib/qless/job_reservers/round_robin.rb +3 -1
  9. data/lib/qless/job_reservers/shuffled_round_robin.rb +14 -0
  10. data/lib/qless/lua_script.rb +42 -0
  11. data/lib/qless/middleware/redis_reconnect.rb +24 -0
  12. data/lib/qless/middleware/retry_exceptions.rb +43 -0
  13. data/lib/qless/middleware/sentry.rb +70 -0
  14. data/lib/qless/qless-core/cancel.lua +89 -59
  15. data/lib/qless/qless-core/complete.lua +16 -1
  16. data/lib/qless/qless-core/config.lua +12 -0
  17. data/lib/qless/qless-core/deregister_workers.lua +12 -0
  18. data/lib/qless/qless-core/fail.lua +24 -14
  19. data/lib/qless/qless-core/heartbeat.lua +2 -1
  20. data/lib/qless/qless-core/pause.lua +18 -0
  21. data/lib/qless/qless-core/pop.lua +24 -3
  22. data/lib/qless/qless-core/put.lua +14 -1
  23. data/lib/qless/qless-core/qless-lib.lua +2354 -0
  24. data/lib/qless/qless-core/qless.lua +1862 -0
  25. data/lib/qless/qless-core/retry.lua +1 -1
  26. data/lib/qless/qless-core/unfail.lua +54 -0
  27. data/lib/qless/qless-core/unpause.lua +12 -0
  28. data/lib/qless/queue.rb +45 -21
  29. data/lib/qless/server.rb +38 -39
  30. data/lib/qless/server/static/css/docs.css +21 -1
  31. data/lib/qless/server/views/_job.erb +5 -5
  32. data/lib/qless/server/views/overview.erb +14 -9
  33. data/lib/qless/subscriber.rb +48 -0
  34. data/lib/qless/version.rb +1 -1
  35. data/lib/qless/wait_until.rb +19 -0
  36. data/lib/qless/worker.rb +243 -33
  37. metadata +49 -30
  38. data/bin/install_phantomjs +0 -7
  39. data/bin/qless-campfire +0 -106
  40. data/bin/qless-growl +0 -99
  41. data/lib/qless/lua.rb +0 -25
@@ -0,0 +1,1862 @@
1
+ local Qless = {
2
+ ns = 'ql:'
3
+ }
4
+
5
+ local QlessQueue = {
6
+ ns = Qless.ns .. 'q:'
7
+ }
8
+ QlessQueue.__index = QlessQueue
9
+
10
+ local QlessWorker = {
11
+ ns = Qless.ns .. 'w:'
12
+ }
13
+ QlessWorker.__index = QlessWorker
14
+
15
+ local QlessJob = {
16
+ ns = Qless.ns .. 'j:'
17
+ }
18
+ QlessJob.__index = QlessJob
19
+
20
+ local QlessRecurringJob = {}
21
+ QlessRecurringJob.__index = QlessRecurringJob
22
+
23
+ Qless.config = {}
24
+
25
+ function table.extend(self, other)
26
+ for i, v in ipairs(other) do
27
+ table.insert(self, v)
28
+ end
29
+ end
30
+
31
+ function Qless.debug(message)
32
+ redis.call('publish', 'debug', tostring(message))
33
+ end
34
+
35
+ function Qless.publish(channel, message)
36
+ redis.call('publish', Qless.ns .. channel, message)
37
+ end
38
+
39
+ function Qless.job(jid)
40
+ assert(jid, 'Job(): no jid provided')
41
+ local job = {}
42
+ setmetatable(job, QlessJob)
43
+ job.jid = jid
44
+ return job
45
+ end
46
+
47
+ function Qless.recurring(jid)
48
+ assert(jid, 'Recurring(): no jid provided')
49
+ local job = {}
50
+ setmetatable(job, QlessRecurringJob)
51
+ job.jid = jid
52
+ return job
53
+ end
54
+
55
+ function Qless.failed(group, start, limit)
56
+ start = assert(tonumber(start or 0),
57
+ 'Failed(): Arg "start" is not a number: ' .. (start or 'nil'))
58
+ limit = assert(tonumber(limit or 25),
59
+ 'Failed(): Arg "limit" is not a number: ' .. (limit or 'nil'))
60
+
61
+ if group then
62
+ return {
63
+ total = redis.call('llen', 'ql:f:' .. group),
64
+ jobs = redis.call('lrange', 'ql:f:' .. group, start, limit - 1)
65
+ }
66
+ else
67
+ local response = {}
68
+ local groups = redis.call('smembers', 'ql:failures')
69
+ for index, group in ipairs(groups) do
70
+ response[group] = redis.call('llen', 'ql:f:' .. group)
71
+ end
72
+ return response
73
+ end
74
+ end
75
+
76
+ function Qless.jobs(now, state, ...)
77
+ assert(state, 'Jobs(): Arg "state" missing')
78
+ if state == 'complete' then
79
+ local offset = assert(tonumber(arg[1] or 0),
80
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[1]))
81
+ local count = assert(tonumber(arg[2] or 25),
82
+ 'Jobs(): Arg "count" not a number: ' .. tostring(arg[2]))
83
+ return redis.call('zrevrange', 'ql:completed', offset,
84
+ offset + count - 1)
85
+ else
86
+ local name = assert(arg[1], 'Jobs(): Arg "queue" missing')
87
+ local offset = assert(tonumber(arg[2] or 0),
88
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[2]))
89
+ local count = assert(tonumber(arg[3] or 25),
90
+ 'Jobs(): Arg "count" not a number: ' .. tostring(arg[3]))
91
+
92
+ local queue = Qless.queue(name)
93
+ if state == 'running' then
94
+ return queue.locks.peek(now, offset, count)
95
+ elseif state == 'stalled' then
96
+ return queue.locks.expired(now, offset, count)
97
+ elseif state == 'scheduled' then
98
+ return queue.scheduled.peek(now, offset, count)
99
+ elseif state == 'depends' then
100
+ return queue.depends.peek(now, offset, count)
101
+ elseif state == 'recurring' then
102
+ return queue.recurring.peek(now, offset, count)
103
+ else
104
+ error('Jobs(): Unknown type "' .. state .. '"')
105
+ end
106
+ end
107
+ end
108
+
109
+ function Qless.track(now, command, jid)
110
+ if command ~= nil then
111
+ assert(jid, 'Track(): Arg "jid" missing')
112
+ if string.lower(ARGV[1]) == 'track' then
113
+ Qless.publish('track', jid)
114
+ return redis.call('zadd', 'ql:tracked', now, jid)
115
+ elseif string.lower(ARGV[1]) == 'untrack' then
116
+ Qless.publish('untrack', jid)
117
+ return redis.call('zrem', 'ql:tracked', jid)
118
+ else
119
+ error('Track(): Unknown action "' .. command .. '"')
120
+ end
121
+ else
122
+ local response = {
123
+ jobs = {},
124
+ expired = {}
125
+ }
126
+ local jids = redis.call('zrange', 'ql:tracked', 0, -1)
127
+ for index, jid in ipairs(jids) do
128
+ local data = Qless.job(jid):data()
129
+ if data then
130
+ table.insert(response.jobs, data)
131
+ else
132
+ table.insert(response.expired, jid)
133
+ end
134
+ end
135
+ return response
136
+ end
137
+ end
138
+
139
+ function Qless.tag(now, command, ...)
140
+ assert(command, 'Tag(): Arg "command" must be "add", "remove", "get" or "top"')
141
+
142
+ if command == 'add' then
143
+ local jid = assert(arg[1], 'Tag(): Arg "jid" missing')
144
+ local tags = redis.call('hget', QlessJob.ns .. jid, 'tags')
145
+ if tags then
146
+ tags = cjson.decode(tags)
147
+ local _tags = {}
148
+ for i,v in ipairs(tags) do _tags[v] = true end
149
+
150
+ for i=2,#arg do
151
+ local tag = arg[i]
152
+ if _tags[tag] == nil then
153
+ table.insert(tags, tag)
154
+ end
155
+ redis.call('zadd', 'ql:t:' .. tag, now, jid)
156
+ redis.call('zincrby', 'ql:tags', 1, tag)
157
+ end
158
+
159
+ tags = cjson.encode(tags)
160
+ redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
161
+ return tags
162
+ else
163
+ return false
164
+ end
165
+ elseif command == 'remove' then
166
+ local jid = assert(arg[1], 'Tag(): Arg "jid" missing')
167
+ local tags = redis.call('hget', QlessJob.ns .. jid, 'tags')
168
+ if tags then
169
+ tags = cjson.decode(tags)
170
+ local _tags = {}
171
+ for i,v in ipairs(tags) do _tags[v] = true end
172
+
173
+ for i=2,#arg do
174
+ local tag = arg[i]
175
+ _tags[tag] = nil
176
+ redis.call('zrem', 'ql:t:' .. tag, jid)
177
+ redis.call('zincrby', 'ql:tags', -1, tag)
178
+ end
179
+
180
+ local results = {}
181
+ for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
182
+
183
+ tags = cjson.encode(results)
184
+ redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
185
+ return results
186
+ else
187
+ return false
188
+ end
189
+ elseif command == 'get' then
190
+ local tag = assert(arg[1], 'Tag(): Arg "tag" missing')
191
+ local offset = assert(tonumber(arg[2] or 0),
192
+ 'Tag(): Arg "offset" not a number: ' .. tostring(arg[2]))
193
+ local count = assert(tonumber(arg[3] or 25),
194
+ 'Tag(): Arg "count" not a number: ' .. tostring(arg[3]))
195
+ return {
196
+ total = redis.call('zcard', 'ql:t:' .. tag),
197
+ jobs = redis.call('zrange', 'ql:t:' .. tag, offset, count)
198
+ }
199
+ elseif command == 'top' then
200
+ local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1]))
201
+ local count = assert(tonumber(arg[2] or 25), 'Tag(): Arg "count" not a number: ' .. tostring(arg[2]))
202
+ return redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, count)
203
+ else
204
+ error('Tag(): First argument must be "add", "remove" or "get"')
205
+ end
206
+ end
207
+
208
+ function Qless.cancel(...)
209
+ local dependents = {}
210
+ for _, jid in ipairs(arg) do
211
+ dependents[jid] = redis.call(
212
+ 'smembers', QlessJob.ns .. jid .. '-dependents') or {}
213
+ end
214
+
215
+ for i, jid in ipairs(arg) do
216
+ for j, dep in ipairs(dependents[jid]) do
217
+ if dependents[dep] == nil then
218
+ error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep ..
219
+ ' but is not mentioned to be canceled')
220
+ end
221
+ end
222
+ end
223
+
224
+ for _, jid in ipairs(arg) do
225
+ local state, queue, failure, worker = unpack(redis.call(
226
+ 'hmget', QlessJob.ns .. jid, 'state', 'queue', 'failure', 'worker'))
227
+
228
+ if state ~= 'complete' then
229
+ local encoded = cjson.encode({
230
+ jid = jid,
231
+ worker = worker,
232
+ event = 'canceled',
233
+ queue = queue
234
+ })
235
+ Qless.publish('log', encoded)
236
+
237
+ if worker and (worker ~= '') then
238
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
239
+ Qless.publish('w:', worker, encoded)
240
+ end
241
+
242
+ if queue then
243
+ local queue = Qless.queue(queue)
244
+ queue.work.remove(jid)
245
+ queue.locks.remove(jid)
246
+ queue.scheduled.remove(jid)
247
+ queue.depends.remove(jid)
248
+ end
249
+
250
+ for i, j in ipairs(redis.call(
251
+ 'smembers', QlessJob.ns .. jid .. '-dependencies')) do
252
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', jid)
253
+ end
254
+
255
+ redis.call('del', QlessJob.ns .. jid .. '-dependencies')
256
+
257
+ if state == 'failed' then
258
+ failure = cjson.decode(failure)
259
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
260
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
261
+ redis.call('srem', 'ql:failures', failure.group)
262
+ end
263
+ end
264
+
265
+ local tags = cjson.decode(
266
+ redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
267
+ for i, tag in ipairs(tags) do
268
+ redis.call('zrem', 'ql:t:' .. tag, jid)
269
+ redis.call('zincrby', 'ql:tags', -1, tag)
270
+ end
271
+
272
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
273
+ Qless.publish('canceled', jid)
274
+ end
275
+
276
+ redis.call('del', QlessJob.ns .. jid)
277
+ redis.call('del', QlessJob.ns .. jid .. '-history')
278
+ end
279
+ end
280
+
281
+ return arg
282
+ end
283
+
284
+
285
+ Qless.config.defaults = {
286
+ ['application'] = 'qless',
287
+ ['heartbeat'] = 60,
288
+ ['grace-period'] = 10,
289
+ ['stats-history'] = 30,
290
+ ['histogram-history'] = 7,
291
+ ['jobs-history-count'] = 50000,
292
+ ['jobs-history'] = 604800
293
+ }
294
+
295
+ Qless.config.get = function(key, default)
296
+ if key then
297
+ return redis.call('hget', 'ql:config', key) or
298
+ Qless.config.defaults[key] or default
299
+ else
300
+ local reply = redis.call('hgetall', 'ql:config')
301
+ for i = 1, #reply, 2 do
302
+ Qless.config.defaults[reply[i]] = reply[i + 1]
303
+ end
304
+ return Qless.config.defaults
305
+ end
306
+ end
307
+
308
+ Qless.config.set = function(option, value)
309
+ assert(option, 'config.set(): Arg "option" missing')
310
+ assert(value , 'config.set(): Arg "value" missing')
311
+ Qless.publish('log', cjson.encode({
312
+ event = 'config_set',
313
+ option = option,
314
+ value = value
315
+ }))
316
+
317
+ redis.call('hset', 'ql:config', option, value)
318
+ end
319
+
320
+ Qless.config.unset = function(option)
321
+ assert(option, 'config.unset(): Arg "option" missing')
322
+ Qless.publish('log', cjson.encode({
323
+ event = 'config_unset',
324
+ option = option
325
+ }))
326
+
327
+ redis.call('hdel', 'ql:config', option)
328
+ end
329
+
330
+ function QlessJob:data(...)
331
+ local job = redis.call(
332
+ 'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue',
333
+ 'worker', 'priority', 'expires', 'retries', 'remaining', 'data',
334
+ 'tags', 'failure')
335
+
336
+ if not job[1] then
337
+ return nil
338
+ end
339
+
340
+ local data = {
341
+ jid = job[1],
342
+ klass = job[2],
343
+ state = job[3],
344
+ queue = job[4],
345
+ worker = job[5] or '',
346
+ tracked = redis.call(
347
+ 'zscore', 'ql:tracked', self.jid) ~= false,
348
+ priority = tonumber(job[6]),
349
+ expires = tonumber(job[7]) or 0,
350
+ retries = tonumber(job[8]),
351
+ remaining = math.floor(tonumber(job[9])),
352
+ data = cjson.decode(job[10]),
353
+ tags = cjson.decode(job[11]),
354
+ history = self:history(),
355
+ failure = cjson.decode(job[12] or '{}'),
356
+ dependents = redis.call(
357
+ 'smembers', QlessJob.ns .. self.jid .. '-dependents'),
358
+ dependencies = redis.call(
359
+ 'smembers', QlessJob.ns .. self.jid .. '-dependencies')
360
+ }
361
+
362
+ if #arg > 0 then
363
+ local response = {}
364
+ for index, key in ipairs(arg) do
365
+ table.insert(response, data[key])
366
+ end
367
+ return response
368
+ else
369
+ return data
370
+ end
371
+ end
372
+
373
+ function QlessJob:complete(now, worker, queue, data, ...)
374
+ assert(worker, 'Complete(): Arg "worker" missing')
375
+ assert(queue , 'Complete(): Arg "queue" missing')
376
+ data = assert(cjson.decode(data),
377
+ 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data))
378
+
379
+ local options = {}
380
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
381
+
382
+ local nextq = options['next']
383
+ local delay = assert(tonumber(options['delay'] or 0))
384
+ local depends = assert(cjson.decode(options['depends'] or '[]'),
385
+ 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends']))
386
+
387
+ if delay > 0 and #depends > 0 then
388
+ error('Complete(): "delay" and "depends" are not allowed together')
389
+ end
390
+
391
+ if options['delay'] and nextq == nil then
392
+ error('Complete(): "delay" cannot be used without a "next".')
393
+ end
394
+
395
+ if options['depends'] and nextq == nil then
396
+ error('Complete(): "depends" cannot be used without a "next".')
397
+ end
398
+
399
+ local bin = now - (now % 86400)
400
+
401
+ local lastworker, state, priority, retries = unpack(
402
+ redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state',
403
+ 'priority', 'retries', 'dependents'))
404
+
405
+ if lastworker == false then
406
+ error('Complete(): Job does not exist')
407
+ elseif (state ~= 'running') then
408
+ error('Complete(): Job is not currently running: ' .. state)
409
+ elseif lastworker ~= worker then
410
+ error('Complete(): Job has been handed out to another worker: ' ..
411
+ tostring(lastworker))
412
+ end
413
+
414
+ self:history(now, 'done')
415
+
416
+ if data then
417
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
418
+ end
419
+
420
+ local queue_obj = Qless.queue(queue)
421
+ queue_obj.work.remove(self.jid)
422
+ queue_obj.locks.remove(self.jid)
423
+ queue_obj.scheduled.remove(self.jid)
424
+
425
+ local waiting = 0
426
+ Qless.queue(queue):stat(now, 'run', waiting)
427
+
428
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
429
+
430
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
431
+ Qless.publish('completed', self.jid)
432
+ end
433
+
434
+ if nextq then
435
+ queue_obj = Qless.queue(nextq)
436
+ Qless.publish('log', cjson.encode({
437
+ jid = self.jid,
438
+ event = 'advanced',
439
+ queue = queue,
440
+ to = nextq
441
+ }))
442
+
443
+ self:history(now, 'put', {q = nextq})
444
+
445
+ if redis.call('zscore', 'ql:queues', nextq) == false then
446
+ redis.call('zadd', 'ql:queues', now, nextq)
447
+ end
448
+
449
+ redis.call('hmset', QlessJob.ns .. self.jid,
450
+ 'state', 'waiting',
451
+ 'worker', '',
452
+ 'failure', '{}',
453
+ 'queue', nextq,
454
+ 'expires', 0,
455
+ 'remaining', tonumber(retries))
456
+
457
+ if delay > 0 then
458
+ queue_obj.scheduled.add(now + delay, self.jid)
459
+ return 'scheduled'
460
+ else
461
+ local count = 0
462
+ for i, j in ipairs(depends) do
463
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
464
+ if (state and state ~= 'complete') then
465
+ count = count + 1
466
+ redis.call(
467
+ 'sadd', QlessJob.ns .. j .. '-dependents',self.jid)
468
+ redis.call(
469
+ 'sadd', QlessJob.ns .. self.jid .. '-dependencies', j)
470
+ end
471
+ end
472
+ if count > 0 then
473
+ queue_obj.depends.add(now, self.jid)
474
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'depends')
475
+ return 'depends'
476
+ else
477
+ queue_obj.work.add(now, priority, self.jid)
478
+ return 'waiting'
479
+ end
480
+ end
481
+ else
482
+ Qless.publish('log', cjson.encode({
483
+ jid = self.jid,
484
+ event = 'completed',
485
+ queue = queue
486
+ }))
487
+
488
+ redis.call('hmset', QlessJob.ns .. self.jid,
489
+ 'state', 'complete',
490
+ 'worker', '',
491
+ 'failure', '{}',
492
+ 'queue', '',
493
+ 'expires', 0,
494
+ 'remaining', tonumber(retries))
495
+
496
+ local count = Qless.config.get('jobs-history-count')
497
+ local time = Qless.config.get('jobs-history')
498
+
499
+ count = tonumber(count or 50000)
500
+ time = tonumber(time or 7 * 24 * 60 * 60)
501
+
502
+ redis.call('zadd', 'ql:completed', now, self.jid)
503
+
504
+ local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
505
+ for index, jid in ipairs(jids) do
506
+ local tags = cjson.decode(redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
507
+ for i, tag in ipairs(tags) do
508
+ redis.call('zrem', 'ql:t:' .. tag, jid)
509
+ redis.call('zincrby', 'ql:tags', -1, tag)
510
+ end
511
+ redis.call('del', QlessJob.ns .. jid)
512
+ redis.call('del', QlessJob.ns .. jid .. '-history')
513
+ end
514
+ redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
515
+
516
+ jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
517
+ for index, jid in ipairs(jids) do
518
+ local tags = cjson.decode(redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
519
+ for i, tag in ipairs(tags) do
520
+ redis.call('zrem', 'ql:t:' .. tag, jid)
521
+ redis.call('zincrby', 'ql:tags', -1, tag)
522
+ end
523
+ redis.call('del', QlessJob.ns .. jid)
524
+ redis.call('del', QlessJob.ns .. jid .. '-history')
525
+ end
526
+ redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
527
+
528
+ for i, j in ipairs(redis.call('smembers', QlessJob.ns .. self.jid .. '-dependents')) do
529
+ redis.call('srem', QlessJob.ns .. j .. '-dependencies', self.jid)
530
+ if redis.call('scard', QlessJob.ns .. j .. '-dependencies') == 0 then
531
+ local q, p = unpack(redis.call('hmget', QlessJob.ns .. j, 'queue', 'priority'))
532
+ if q then
533
+ local queue = Qless.queue(q)
534
+ queue.depends.remove(j)
535
+ queue.work.add(now, p, j)
536
+ redis.call('hset', QlessJob.ns .. j, 'state', 'waiting')
537
+ end
538
+ end
539
+ end
540
+
541
+ redis.call('del', QlessJob.ns .. self.jid .. '-dependents')
542
+
543
+ return 'complete'
544
+ end
545
+ end
546
+
547
+ function QlessJob:fail(now, worker, group, message, data)
548
+ local worker = assert(worker , 'Fail(): Arg "worker" missing')
549
+ local group = assert(group , 'Fail(): Arg "group" missing')
550
+ local message = assert(message , 'Fail(): Arg "message" missing')
551
+
552
+ local bin = now - (now % 86400)
553
+
554
+ if data then
555
+ data = cjson.decode(data)
556
+ end
557
+
558
+ local queue, state = unpack(redis.call('hmget', QlessJob.ns .. self.jid,
559
+ 'queue', 'state'))
560
+
561
+ if state ~= 'running' then
562
+ error('Fail(): Job not currently running: ' .. state)
563
+ end
564
+
565
+ Qless.publish('log', cjson.encode({
566
+ jid = self.jid,
567
+ event = 'failed',
568
+ worker = worker,
569
+ group = group,
570
+ message = message
571
+ }))
572
+
573
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
574
+ Qless.publish('failed', self.jid)
575
+ end
576
+
577
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
578
+
579
+ self:history(now, 'failed', {worker = worker, group = group})
580
+
581
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failures', 1)
582
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed' , 1)
583
+
584
+ local queue_obj = Qless.queue(queue)
585
+ queue_obj.work.remove(self.jid)
586
+ queue_obj.locks.remove(self.jid)
587
+ queue_obj.scheduled.remove(self.jid)
588
+
589
+ if data then
590
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
591
+ end
592
+
593
+ redis.call('hmset', QlessJob.ns .. self.jid,
594
+ 'state', 'failed',
595
+ 'worker', '',
596
+ 'expires', '',
597
+ 'failure', cjson.encode({
598
+ ['group'] = group,
599
+ ['message'] = message,
600
+ ['when'] = math.floor(now),
601
+ ['worker'] = worker
602
+ }))
603
+
604
+ redis.call('sadd', 'ql:failures', group)
605
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
606
+
607
+
608
+ return self.jid
609
+ end
610
+
611
+ function QlessJob:retry(now, queue, worker, delay, group, message)
612
+ assert(queue , 'Retry(): Arg "queue" missing')
613
+ assert(worker, 'Retry(): Arg "worker" missing')
614
+ delay = assert(tonumber(delay or 0),
615
+ 'Retry(): Arg "delay" not a number: ' .. tostring(delay))
616
+
617
+ local oldqueue, state, retries, oldworker, priority, failure = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state', 'retries', 'worker', 'priority', 'failure'))
618
+
619
+ if oldworker == false then
620
+ error('Retry(): Job does not exist')
621
+ elseif state ~= 'running' then
622
+ error('Retry(): Job is not currently running: ' .. state)
623
+ elseif oldworker ~= worker then
624
+ error('Retry(): Job has been handed out to another worker: ' .. oldworker)
625
+ end
626
+
627
+ local remaining = tonumber(redis.call(
628
+ 'hincrbyfloat', QlessJob.ns .. self.jid, 'remaining', -0.5))
629
+ if (remaining * 2) % 2 == 1 then
630
+ local remaining = tonumber(redis.call(
631
+ 'hincrbyfloat', QlessJob.ns .. self.jid, 'remaining', -0.5))
632
+ end
633
+
634
+ Qless.queue(oldqueue).locks.remove(self.jid)
635
+
636
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
637
+
638
+ if remaining < 0 then
639
+ local group = 'failed-retries-' .. queue
640
+ self:history(now, 'failed', {group = group})
641
+
642
+ redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed',
643
+ 'worker', '',
644
+ 'expires', '')
645
+ if failure == {} then
646
+ redis.call('hset', QlessJob.ns .. self.jid,
647
+ 'failure', cjson.encode({
648
+ ['group'] = group,
649
+ ['message'] =
650
+ 'Job exhausted retries in queue "' .. self.name .. '"',
651
+ ['when'] = now,
652
+ ['worker'] = unpack(job:data('worker'))
653
+ }))
654
+ end
655
+
656
+ redis.call('sadd', 'ql:failures', group)
657
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
658
+ else
659
+ local queue_obj = Qless.queue(queue)
660
+ if delay > 0 then
661
+ queue_obj.scheduled.add(now + delay, self.jid)
662
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'scheduled')
663
+ else
664
+ queue_obj.work.add(now, priority, self.jid)
665
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting')
666
+ end
667
+
668
+ if group ~= nil and message ~= nil then
669
+ redis.call('hset', QlessJob.ns .. self.jid,
670
+ 'failure', cjson.encode({
671
+ ['group'] = group,
672
+ ['message'] = message,
673
+ ['when'] = math.floor(now),
674
+ ['worker'] = worker
675
+ })
676
+ )
677
+ end
678
+ end
679
+
680
+ return math.floor(remaining)
681
+ end
682
+
683
+ function QlessJob:depends(now, command, ...)
684
+ assert(command, 'Depends(): Arg "command" missing')
685
+ if redis.call('hget', QlessJob.ns .. self.jid, 'state') ~= 'depends' then
686
+ return false
687
+ end
688
+
689
+ if command == 'on' then
690
+ for i, j in ipairs(arg) do
691
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
692
+ if (state and state ~= 'complete') then
693
+ redis.call('sadd', QlessJob.ns .. j .. '-dependents' , self.jid)
694
+ redis.call('sadd', QlessJob.ns .. self.jid .. '-dependencies', j)
695
+ end
696
+ end
697
+ return true
698
+ elseif command == 'off' then
699
+ if arg[1] == 'all' then
700
+ for i, j in ipairs(redis.call('smembers', QlessJob.ns .. self.jid .. '-dependencies')) do
701
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid)
702
+ end
703
+ redis.call('del', QlessJob.ns .. self.jid .. '-dependencies')
704
+ local q, p = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'priority'))
705
+ if q then
706
+ local queue_obj = Qless.queue(q)
707
+ queue_obj.depends.remove(self.jid)
708
+ queue_obj.work.add(now, p, self.jid)
709
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting')
710
+ end
711
+ else
712
+ for i, j in ipairs(arg) do
713
+ redis.call('srem', QlessJob.ns .. j .. '-dependents', self.jid)
714
+ redis.call('srem', QlessJob.ns .. self.jid .. '-dependencies', j)
715
+ if redis.call('scard', QlessJob.ns .. self.jid .. '-dependencies') == 0 then
716
+ local q, p = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'priority'))
717
+ if q then
718
+ local queue_obj = Qless.queue(q)
719
+ queue_obj.depends.remove(self.jid)
720
+ queue_obj.work.add(now, p, self.jid)
721
+ redis.call('hset', QlessJob.ns .. self.jid, 'state', 'waiting')
722
+ end
723
+ end
724
+ end
725
+ end
726
+ return true
727
+ else
728
+ error('Depends(): Argument "command" must be "on" or "off"')
729
+ end
730
+ end
731
+
732
+ function QlessJob:heartbeat(now, worker, data)
733
+ assert(worker, 'Heatbeat(): Arg "worker" missing')
734
+
735
+ local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue') or ''
736
+ local expires = now + tonumber(
737
+ Qless.config.get(queue .. '-heartbeat') or
738
+ Qless.config.get('heartbeat', 60))
739
+
740
+ if data then
741
+ data = cjson.decode(data)
742
+ end
743
+
744
+ local job_worker, state = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state'))
745
+ if job_worker == false then
746
+ error('Heartbeat(): Job does not exist')
747
+ elseif state ~= 'running' then
748
+ error('Heartbeat(): Job not currently running: ' .. state)
749
+ elseif job_worker ~= worker or #job_worker == 0 then
750
+ error('Heartbeat(): Job has been handed out to another worker: ' .. job_worker)
751
+ else
752
+ if data then
753
+ redis.call('hmset', QlessJob.ns .. self.jid, 'expires', expires, 'worker', worker, 'data', cjson.encode(data))
754
+ else
755
+ redis.call('hmset', QlessJob.ns .. self.jid, 'expires', expires, 'worker', worker)
756
+ end
757
+
758
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
759
+
760
+ local queue = Qless.queue(redis.call('hget', QlessJob.ns .. self.jid, 'queue'))
761
+ queue.locks.add(expires, self.jid)
762
+ return expires
763
+ end
764
+ end
765
+
766
+ function QlessJob:priority(priority)
767
+ priority = assert(tonumber(priority),
768
+ 'Priority(): Arg "priority" missing or not a number: ' .. tostring(priority))
769
+
770
+ local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue')
771
+
772
+ if queue == nil then
773
+ return false
774
+ elseif queue == '' then
775
+ redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority)
776
+ return priority
777
+ else
778
+ local queue_obj = Qless.queue(queue)
779
+ if queue_obj.work.score(self.jid) then
780
+ queue_obj.work.add(0, priority, self.jid)
781
+ end
782
+ redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority)
783
+ return priority
784
+ end
785
+ end
786
+
787
+ function QlessJob:update(data)
788
+ local tmp = {}
789
+ for k, v in pairs(data) do
790
+ table.insert(tmp, k)
791
+ table.insert(tmp, v)
792
+ end
793
+ redis.call('hmset', QlessJob.ns .. self.jid, unpack(tmp))
794
+ end
795
+
796
+ function QlessJob:timeout(now)
797
+ local queue_name, state, worker = unpack(redis.call('hmget',
798
+ QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
799
+ if queue_name == nil then
800
+ error('Timeout(): Job does not exist')
801
+ elseif state ~= 'running' then
802
+ error('Timeout(): Job not running')
803
+ else
804
+ self:history(now, 'timed-out')
805
+ local queue = Qless.queue(queue_name)
806
+ queue.locks.remove(self.jid)
807
+ queue.work.add(now, math.huge, self.jid)
808
+ redis.call('hmset', QlessJob.ns .. self.jid,
809
+ 'state', 'stalled', 'expires', 0)
810
+ local encoded = cjson.encode({
811
+ jid = self.jid,
812
+ event = 'lock_lost',
813
+ worker = worker
814
+ })
815
+ Qless.publish('w:' .. worker, encoded)
816
+ Qless.publish('log', encoded)
817
+ return queue
818
+ end
819
+ end
820
+
821
+ function QlessJob:history(now, what, item)
822
+ local history = redis.call('hget', QlessJob.ns .. self.jid, 'history')
823
+ if history then
824
+ history = cjson.decode(history)
825
+ for i, value in ipairs(history) do
826
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
827
+ cjson.encode({math.floor(value.put), 'put', {q = value.q}}))
828
+
829
+ if value.popped then
830
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
831
+ cjson.encode({math.floor(value.popped), 'popped',
832
+ {worker = value.worker}}))
833
+ end
834
+
835
+ if value.failed then
836
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
837
+ cjson.encode(
838
+ {math.floor(value.failed), 'failed', nil}))
839
+ end
840
+
841
+ if value.done then
842
+ redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
843
+ cjson.encode(
844
+ {math.floor(value.done), 'done', nil}))
845
+ end
846
+ end
847
+ redis.call('hdel', QlessJob.ns .. self.jid, 'history')
848
+ end
849
+
850
+ if what == nil then
851
+ local response = {}
852
+ for i, value in ipairs(redis.call('lrange',
853
+ QlessJob.ns .. self.jid .. '-history', 0, -1)) do
854
+ value = cjson.decode(value)
855
+ local dict = value[3] or {}
856
+ dict['when'] = value[1]
857
+ dict['what'] = value[2]
858
+ table.insert(response, dict)
859
+ end
860
+ return response
861
+ else
862
+ return redis.call('rpush', QlessJob.ns .. self.jid .. '-history',
863
+ cjson.encode({math.floor(now), what, item}))
864
+ end
865
+ end
866
+ function Qless.queue(name)
867
+ assert(name, 'Queue(): no queue name provided')
868
+ local queue = {}
869
+ setmetatable(queue, QlessQueue)
870
+ queue.name = name
871
+
872
+ queue.work = {
873
+ peek = function(count)
874
+ if count == 0 then
875
+ return {}
876
+ end
877
+ local jids = {}
878
+ for index, jid in ipairs(redis.call(
879
+ 'zrevrange', queue:prefix('work'), 0, count - 1)) do
880
+ table.insert(jids, jid)
881
+ end
882
+ return jids
883
+ end, remove = function(...)
884
+ if #arg > 0 then
885
+ return redis.call('zrem', queue:prefix('work'), unpack(arg))
886
+ end
887
+ end, add = function(now, priority, jid)
888
+ return redis.call('zadd',
889
+ queue:prefix('work'), priority - (now / 10000000000), jid)
890
+ end, score = function(jid)
891
+ return redis.call('zscore', queue:prefix('work'), jid)
892
+ end, length = function()
893
+ return redis.call('zcard', queue:prefix('work'))
894
+ end
895
+ }
896
+
897
+ queue.locks = {
898
+ expired = function(now, offset, count)
899
+ return redis.call('zrangebyscore',
900
+ queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count)
901
+ end, peek = function(now, offset, count)
902
+ return redis.call('zrangebyscore', queue:prefix('locks'),
903
+ now, math.huge, 'LIMIT', offset, count)
904
+ end, add = function(expires, jid)
905
+ redis.call('zadd', queue:prefix('locks'), expires, jid)
906
+ end, remove = function(...)
907
+ if #arg > 0 then
908
+ return redis.call('zrem', queue:prefix('locks'), unpack(arg))
909
+ end
910
+ end, running = function(now)
911
+ return redis.call('zcount', queue:prefix('locks'), now, math.huge)
912
+ end, length = function(now)
913
+ if now then
914
+ return redis.call('zcount', queue:prefix('locks'), 0, now)
915
+ else
916
+ return redis.call('zcard', queue:prefix('locks'))
917
+ end
918
+ end
919
+ }
920
+
921
+ queue.depends = {
922
+ peek = function(now, offset, count)
923
+ return redis.call('zrange',
924
+ queue:prefix('depends'), offset, offset + count - 1)
925
+ end, add = function(now, jid)
926
+ redis.call('zadd', queue:prefix('depends'), now, jid)
927
+ end, remove = function(...)
928
+ if #arg > 0 then
929
+ return redis.call('zrem', queue:prefix('depends'), unpack(arg))
930
+ end
931
+ end, length = function()
932
+ return redis.call('zcard', queue:prefix('depends'))
933
+ end
934
+ }
935
+
936
+ queue.scheduled = {
937
+ peek = function(now, offset, count)
938
+ return redis.call('zrange',
939
+ queue:prefix('scheduled'), offset, offset + count - 1)
940
+ end, ready = function(now, offset, count)
941
+ return redis.call('zrangebyscore',
942
+ queue:prefix('scheduled'), 0, now, 'LIMIT', offset, count)
943
+ end, add = function(when, jid)
944
+ redis.call('zadd', queue:prefix('scheduled'), when, jid)
945
+ end, remove = function(...)
946
+ if #arg > 0 then
947
+ return redis.call('zrem', queue:prefix('scheduled'), unpack(arg))
948
+ end
949
+ end, length = function()
950
+ return redis.call('zcard', queue:prefix('scheduled'))
951
+ end
952
+ }
953
+
954
+ queue.recurring = {
955
+ peek = function(now, offset, count)
956
+ return redis.call('zrangebyscore', queue:prefix('recur'),
957
+ 0, now, 'LIMIT', offset, count)
958
+ end, ready = function(now, offset, count)
959
+ end, add = function(when, jid)
960
+ redis.call('zadd', queue:prefix('recur'), when, jid)
961
+ end, remove = function(...)
962
+ if #arg > 0 then
963
+ return redis.call('zrem', queue:prefix('recur'), unpack(arg))
964
+ end
965
+ end, update = function(increment, jid)
966
+ redis.call('zincrby', queue:prefix('recur'), increment, jid)
967
+ end, score = function(jid)
968
+ return redis.call('zscore', queue:prefix('recur'), jid)
969
+ end, length = function()
970
+ return redis.call('zcard', queue:prefix('recur'))
971
+ end
972
+ }
973
+ return queue
974
+ end
975
+
976
+ function QlessQueue:prefix(group)
977
+ if group then
978
+ return QlessQueue.ns..self.name..'-'..group
979
+ else
980
+ return QlessQueue.ns..self.name
981
+ end
982
+ end
983
+
984
+ function QlessQueue:stats(now, date)
985
+ date = assert(tonumber(date),
986
+ 'Stats(): Arg "date" missing or not a number: '.. (date or 'nil'))
987
+
988
+ local bin = date - (date % 86400)
989
+
990
+ local histokeys = {
991
+ 's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59',
992
+ 'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59',
993
+ 'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23',
994
+ 'd1','d2','d3','d4','d5','d6'
995
+ }
996
+
997
+ local mkstats = function(name, bin, queue)
998
+ local results = {}
999
+
1000
+ local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
1001
+ local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
1002
+
1003
+ count = tonumber(count) or 0
1004
+ mean = tonumber(mean) or 0
1005
+ vk = tonumber(vk)
1006
+
1007
+ results.count = count or 0
1008
+ results.mean = mean or 0
1009
+ results.histogram = {}
1010
+
1011
+ if not count then
1012
+ results.std = 0
1013
+ else
1014
+ if count > 1 then
1015
+ results.std = math.sqrt(vk / (count - 1))
1016
+ else
1017
+ results.std = 0
1018
+ end
1019
+ end
1020
+
1021
+ local histogram = redis.call('hmget', key, unpack(histokeys))
1022
+ for i=1,#histokeys do
1023
+ table.insert(results.histogram, tonumber(histogram[i]) or 0)
1024
+ end
1025
+ return results
1026
+ end
1027
+
1028
+ local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 'failed', 'failures'))
1029
+ return {
1030
+ retries = tonumber(retries or 0),
1031
+ failed = tonumber(failed or 0),
1032
+ failures = tonumber(failures or 0),
1033
+ wait = mkstats('wait', bin, self.name),
1034
+ run = mkstats('run' , bin, self.name)
1035
+ }
1036
+ end
1037
+
1038
+ function QlessQueue:peek(now, count)
1039
+ count = assert(tonumber(count),
1040
+ 'Peek(): Arg "count" missing or not a number: ' .. tostring(count))
1041
+
1042
+ local jids = self.locks.expired(now, 0, count)
1043
+
1044
+ self:check_recurring(now, count - #jids)
1045
+
1046
+ self:check_scheduled(now, count - #jids)
1047
+
1048
+ table.extend(jids, self.work.peek(count - #jids))
1049
+
1050
+ return jids
1051
+ end
1052
+
1053
+ function QlessQueue:paused()
1054
+ return redis.call('sismember', 'ql:paused_queues', self.name) == 1
1055
+ end
1056
+
1057
+ function QlessQueue.pause(...)
1058
+ redis.call('sadd', 'ql:paused_queues', unpack(arg))
1059
+ end
1060
+
1061
+ function QlessQueue.unpause(...)
1062
+ redis.call('srem', 'ql:paused_queues', unpack(arg))
1063
+ end
1064
+
1065
+ function QlessQueue:pop(now, worker, count)
1066
+ assert(worker, 'Pop(): Arg "worker" missing')
1067
+ count = assert(tonumber(count),
1068
+ 'Pop(): Arg "count" missing or not a number: ' .. tostring(count))
1069
+
1070
+ local expires = now + tonumber(
1071
+ Qless.config.get(self.name .. '-heartbeat') or
1072
+ Qless.config.get('heartbeat', 60))
1073
+
1074
+ if self:paused() then
1075
+ return {}
1076
+ end
1077
+
1078
+ redis.call('zadd', 'ql:workers', now, worker)
1079
+
1080
+ local max_concurrency = tonumber(
1081
+ Qless.config.get(self.name .. '-max-concurrency', 0))
1082
+
1083
+ if max_concurrency > 0 then
1084
+ local allowed = math.max(0, max_concurrency - self.locks.running(now))
1085
+ count = math.min(allowed, count)
1086
+ if count == 0 then
1087
+ return {}
1088
+ end
1089
+ end
1090
+
1091
+ local jids = self:invalidate_locks(now, count)
1092
+
1093
+ self:check_recurring(now, count - #jids)
1094
+
1095
+ self:check_scheduled(now, count - #jids)
1096
+
1097
+ table.extend(jids, self.work.peek(count - #jids))
1098
+
1099
+ local state
1100
+ for index, jid in ipairs(jids) do
1101
+ local job = Qless.job(jid)
1102
+ state = unpack(job:data('state'))
1103
+ job:history(now, 'popped', {worker = worker})
1104
+
1105
+ local waiting = 0
1106
+ self:stat(now, 'wait', waiting)
1107
+
1108
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
1109
+
1110
+ job:update({
1111
+ worker = worker,
1112
+ expires = expires,
1113
+ state = 'running'
1114
+ })
1115
+
1116
+ self.locks.add(expires, jid)
1117
+
1118
+ local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
1119
+ if tracked then
1120
+ Qless.publish('popped', jid)
1121
+ end
1122
+ end
1123
+
1124
+ self.work.remove(unpack(jids))
1125
+
1126
+ return jids
1127
+ end
1128
+
1129
+ function QlessQueue:stat(now, stat, val)
1130
+ local bin = now - (now % 86400)
1131
+ local key = 'ql:s:' .. stat .. ':' .. bin .. ':' .. self.name
1132
+
1133
+ local count, mean, vk = unpack(
1134
+ redis.call('hmget', key, 'total', 'mean', 'vk'))
1135
+
1136
+ count = count or 0
1137
+ if count == 0 then
1138
+ mean = val
1139
+ vk = 0
1140
+ count = 1
1141
+ else
1142
+ count = count + 1
1143
+ local oldmean = mean
1144
+ mean = mean + (val - mean) / count
1145
+ vk = vk + (val - mean) * (val - oldmean)
1146
+ end
1147
+
1148
+ val = math.floor(val)
1149
+ if val < 60 then -- seconds
1150
+ redis.call('hincrby', key, 's' .. val, 1)
1151
+ elseif val < 3600 then -- minutes
1152
+ redis.call('hincrby', key, 'm' .. math.floor(val / 60), 1)
1153
+ elseif val < 86400 then -- hours
1154
+ redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
1155
+ else -- days
1156
+ redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
1157
+ end
1158
+ redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
1159
+ end
1160
+
1161
+ function QlessQueue:put(now, jid, klass, data, delay, ...)
1162
+ assert(jid , 'Put(): Arg "jid" missing')
1163
+ assert(klass, 'Put(): Arg "klass" missing')
1164
+ data = assert(cjson.decode(data),
1165
+ 'Put(): Arg "data" missing or not JSON: ' .. tostring(data))
1166
+ delay = assert(tonumber(delay),
1167
+ 'Put(): Arg "delay" not a number: ' .. tostring(delay))
1168
+
1169
+ local options = {}
1170
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
1171
+
1172
+ local job = Qless.job(jid)
1173
+ local priority, tags, oldqueue, state, failure, retries, worker = unpack(redis.call('hmget', QlessJob.ns .. jid, 'priority', 'tags', 'queue', 'state', 'failure', 'retries', 'worker'))
1174
+
1175
+ retries = assert(tonumber(options['retries'] or retries or 5) , 'Put(): Arg "retries" not a number: ' .. tostring(options['retries']))
1176
+ tags = assert(cjson.decode(options['tags'] or tags or '[]' ), 'Put(): Arg "tags" not JSON' .. tostring(options['tags']))
1177
+ priority = assert(tonumber(options['priority'] or priority or 0), 'Put(): Arg "priority" not a number' .. tostring(options['priority']))
1178
+ local depends = assert(cjson.decode(options['depends'] or '[]') , 'Put(): Arg "depends" not JSON: ' .. tostring(options['depends']))
1179
+
1180
+ if delay > 0 and #depends > 0 then
1181
+ error('Put(): "delay" and "depends" are not allowed to be used together')
1182
+ end
1183
+
1184
+ Qless.publish('log', cjson.encode({
1185
+ jid = jid,
1186
+ event = 'put',
1187
+ queue = self.name
1188
+ }))
1189
+
1190
+ job:history(now, 'put', {q = self.name})
1191
+
1192
+ if oldqueue then
1193
+ local queue_obj = Qless.queue(oldqueue)
1194
+ queue_obj.work.remove(jid)
1195
+ queue_obj.locks.remove(jid)
1196
+ queue_obj.depends.remove(jid)
1197
+ queue_obj.scheduled.remove(jid)
1198
+ end
1199
+
1200
+ if worker then
1201
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
1202
+ Qless.publish('w:' .. worker, cjson.encode({
1203
+ jid = jid,
1204
+ event = 'put',
1205
+ queue = self.name
1206
+ }))
1207
+ end
1208
+
1209
+ if state == 'complete' then
1210
+ redis.call('zrem', 'ql:completed', jid)
1211
+ end
1212
+
1213
+ for i, tag in ipairs(tags) do
1214
+ redis.call('zadd', 'ql:t:' .. tag, now, jid)
1215
+ redis.call('zincrby', 'ql:tags', 1, tag)
1216
+ end
1217
+
1218
+ if state == 'failed' then
1219
+ failure = cjson.decode(failure)
1220
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
1221
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
1222
+ redis.call('srem', 'ql:failures', failure.group)
1223
+ end
1224
+ local bin = failure.when - (failure.when % 86400)
1225
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , -1)
1226
+ end
1227
+
1228
+ redis.call('hmset', QlessJob.ns .. jid,
1229
+ 'jid' , jid,
1230
+ 'klass' , klass,
1231
+ 'data' , cjson.encode(data),
1232
+ 'priority' , priority,
1233
+ 'tags' , cjson.encode(tags),
1234
+ 'state' , ((delay > 0) and 'scheduled') or 'waiting',
1235
+ 'worker' , '',
1236
+ 'expires' , 0,
1237
+ 'queue' , self.name,
1238
+ 'retries' , retries,
1239
+ 'remaining', retries)
1240
+
1241
+ for i, j in ipairs(depends) do
1242
+ local state = redis.call('hget', QlessJob.ns .. j, 'state')
1243
+ if (state and state ~= 'complete') then
1244
+ redis.call('sadd', QlessJob.ns .. j .. '-dependents' , jid)
1245
+ redis.call('sadd', QlessJob.ns .. jid .. '-dependencies', j)
1246
+ end
1247
+ end
1248
+
1249
+ if delay > 0 then
1250
+ self.scheduled.add(now + delay, jid)
1251
+ else
1252
+ if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
1253
+ self.depends.add(now, jid)
1254
+ redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
1255
+ else
1256
+ self.work.add(now, priority, jid)
1257
+ end
1258
+ end
1259
+
1260
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1261
+ redis.call('zadd', 'ql:queues', now, self.name)
1262
+ end
1263
+
1264
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1265
+ Qless.publish('put', jid)
1266
+ end
1267
+
1268
+ return jid
1269
+ end
1270
+
1271
+ function QlessQueue:unfail(now, group, count)
1272
+ assert(group, 'Unfail(): Arg "group" missing')
1273
+ count = assert(tonumber(count or 25),
1274
+ 'Unfail(): Arg "count" not a number: ' .. tostring(count))
1275
+
1276
+ local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1)
1277
+
1278
+ local toinsert = {}
1279
+ for index, jid in ipairs(jids) do
1280
+ local job = Qless.job(job)
1281
+ local data = job:data()
1282
+ job:history(now, 'put', {q = self.name})
1283
+ redis.call('hmset', QlessJob.ns .. data.jid,
1284
+ 'state' , 'waiting',
1285
+ 'worker' , '',
1286
+ 'expires' , 0,
1287
+ 'queue' , self.name,
1288
+ 'remaining', data.retries or 5)
1289
+ self.work.add(now, data.priority, data.jid)
1290
+ end
1291
+
1292
+ redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1)
1293
+ if (redis.call('llen', 'ql:f:' .. group) == 0) then
1294
+ redis.call('srem', 'ql:failures', group)
1295
+ end
1296
+
1297
+ return #jids
1298
+ end
1299
+
1300
+ function QlessQueue:recur(now, jid, klass, data, spec, ...)
1301
+ assert(jid , 'RecurringJob On(): Arg "jid" missing')
1302
+ assert(klass, 'RecurringJob On(): Arg "klass" missing')
1303
+ assert(spec , 'RecurringJob On(): Arg "spec" missing')
1304
+ data = assert(cjson.decode(data),
1305
+ 'RecurringJob On(): Arg "data" not JSON: ' .. tostring(data))
1306
+
1307
+ if spec == 'interval' then
1308
+ local interval = assert(tonumber(arg[1]),
1309
+ 'Recur(): Arg "interval" not a number: ' .. tostring(arg[1]))
1310
+ local offset = assert(tonumber(arg[2]),
1311
+ 'Recur(): Arg "offset" not a number: ' .. tostring(arg[2]))
1312
+ if interval <= 0 then
1313
+ error('Recur(): Arg "interval" must be greater than or equal to 0')
1314
+ end
1315
+
1316
+ local options = {}
1317
+ for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end
1318
+ options.tags = assert(cjson.decode(options.tags or {}),
1319
+ 'Recur(): Arg "tags" must be JSON string array: ' .. tostring(
1320
+ options.tags))
1321
+ options.priority = assert(tonumber(options.priority or 0),
1322
+ 'Recur(): Arg "priority" not a number: ' .. tostring(
1323
+ options.priority))
1324
+ options.retries = assert(tonumber(options.retries or 0),
1325
+ 'Recur(): Arg "retries" not a number: ' .. tostring(
1326
+ options.retries))
1327
+ options.backlog = assert(tonumber(options.backlog or 0),
1328
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(
1329
+ options.backlog))
1330
+
1331
+ local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
1332
+ count = count or 0
1333
+
1334
+ if old_queue then
1335
+ Qless.queue(old_queue).recurring.remove(jid)
1336
+ end
1337
+
1338
+ redis.call('hmset', 'ql:r:' .. jid,
1339
+ 'jid' , jid,
1340
+ 'klass' , klass,
1341
+ 'data' , cjson.encode(data),
1342
+ 'priority', options.priority,
1343
+ 'tags' , cjson.encode(options.tags or {}),
1344
+ 'state' , 'recur',
1345
+ 'queue' , self.name,
1346
+ 'type' , 'interval',
1347
+ 'count' , count,
1348
+ 'interval', interval,
1349
+ 'retries' , options.retries,
1350
+ 'backlog' , options.backlog)
1351
+ self.recurring.add(now + offset, jid)
1352
+
1353
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1354
+ redis.call('zadd', 'ql:queues', now, self.name)
1355
+ end
1356
+
1357
+ return jid
1358
+ else
1359
+ error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
1360
+ end
1361
+ end
1362
+
1363
+ function QlessQueue:length()
1364
+ return self.locks.length() + self.work.length() + self.scheduled.length()
1365
+ end
1366
+
1367
+ function QlessQueue:check_recurring(now, count)
1368
+ local moved = 0
1369
+ local r = self.recurring.peek(now, 0, count)
1370
+ for index, jid in ipairs(r) do
1371
+ local klass, data, priority, tags, retries, interval, backlog = unpack(
1372
+ redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
1373
+ 'tags', 'retries', 'interval', 'backlog'))
1374
+ local _tags = cjson.decode(tags)
1375
+ local score = math.floor(tonumber(self.recurring.score(jid)))
1376
+ interval = tonumber(interval)
1377
+
1378
+ backlog = tonumber(backlog or 0)
1379
+ if backlog ~= 0 then
1380
+ local num = ((now - score) / interval)
1381
+ if num > backlog then
1382
+ score = score + (
1383
+ math.ceil(num - backlog) * interval
1384
+ )
1385
+ end
1386
+ end
1387
+
1388
+ while (score <= now) and (moved < count) do
1389
+ local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
1390
+ moved = moved + 1
1391
+
1392
+ for i, tag in ipairs(_tags) do
1393
+ redis.call('zadd', 'ql:t:' .. tag, now, jid .. '-' .. count)
1394
+ redis.call('zincrby', 'ql:tags', 1, tag)
1395
+ end
1396
+
1397
+ local child_jid = jid .. '-' .. count
1398
+ redis.call('hmset', QlessJob.ns .. child_jid,
1399
+ 'jid' , jid .. '-' .. count,
1400
+ 'klass' , klass,
1401
+ 'data' , data,
1402
+ 'priority' , priority,
1403
+ 'tags' , tags,
1404
+ 'state' , 'waiting',
1405
+ 'worker' , '',
1406
+ 'expires' , 0,
1407
+ 'queue' , self.name,
1408
+ 'retries' , retries,
1409
+ 'remaining', retries)
1410
+ Qless.job(child_jid):history(score, 'put', {q = self.name})
1411
+
1412
+ self.work.add(score, priority, jid .. '-' .. count)
1413
+
1414
+ score = score + interval
1415
+ self.recurring.add(score, jid)
1416
+ end
1417
+ end
1418
+ end
1419
+
1420
+ function QlessQueue:check_scheduled(now, count, execute)
1421
+ local zadd = {}
1422
+ local scheduled = self.scheduled.ready(now, 0, count)
1423
+ for index, jid in ipairs(scheduled) do
1424
+ local priority = tonumber(
1425
+ redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
1426
+ self.work.add(now, priority, jid)
1427
+
1428
+ redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
1429
+ end
1430
+
1431
+ if #zadd > 0 then
1432
+ self.scheduled.remove(unpack(scheduled))
1433
+ end
1434
+ end
1435
+
1436
+ function QlessQueue:invalidate_locks(now, count)
1437
+ local jids = {}
1438
+ for index, jid in ipairs(self.locks.expired(now, 0, count)) do
1439
+ local worker, failure = unpack(
1440
+ redis.call('hmget', QlessJob.ns .. jid, 'worker', 'failure'))
1441
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
1442
+
1443
+ local grace_period = tonumber(Qless.config.get('grace-period'))
1444
+
1445
+ local remaining = tonumber(redis.call(
1446
+ 'hincrbyfloat', QlessJob.ns .. jid, 'remaining', -0.5))
1447
+
1448
+ local send_message = ((remaining * 2) % 2 == 1)
1449
+ local invalidate = not send_message
1450
+
1451
+ if grace_period <= 0 then
1452
+ remaining = tonumber(redis.call(
1453
+ 'hincrbyfloat', QlessJob.ns .. jid, 'remaining', -0.5))
1454
+ send_message = true
1455
+ invalidate = true
1456
+ end
1457
+
1458
+ if send_message then
1459
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1460
+ Qless.publish('stalled', jid)
1461
+ end
1462
+ Qless.job(jid):history(now, 'timed-out')
1463
+
1464
+ local encoded = cjson.encode({
1465
+ jid = jid,
1466
+ event = 'lock_lost',
1467
+ worker = worker
1468
+ })
1469
+ Qless.publish('w:' .. worker, encoded)
1470
+ Qless.publish('log', encoded)
1471
+ self.locks.add(now + grace_period, jid)
1472
+
1473
+ local bin = now - (now % 86400)
1474
+ redis.call('hincrby',
1475
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 1)
1476
+ end
1477
+
1478
+ if invalidate then
1479
+ if remaining < 0 then
1480
+ self.work.remove(jid)
1481
+ self.locks.remove(jid)
1482
+ self.scheduled.remove(jid)
1483
+
1484
+ local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
1485
+ local job = Qless.job(jid)
1486
+ job:history(now, 'failed', {group = group})
1487
+ redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed',
1488
+ 'worker', '',
1489
+ 'expires', '')
1490
+ if failure == {} then
1491
+ redis.call('hset', QlessJob.ns .. jid,
1492
+ 'failure', cjson.encode({
1493
+ ['group'] = group,
1494
+ ['message'] =
1495
+ 'Job exhausted retries in queue "' .. self.name .. '"',
1496
+ ['when'] = now,
1497
+ ['worker'] = unpack(job:data('worker'))
1498
+ }))
1499
+ end
1500
+
1501
+ redis.call('sadd', 'ql:failures', group)
1502
+ redis.call('lpush', 'ql:f:' .. group, jid)
1503
+
1504
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1505
+ Qless.publish('failed', jid)
1506
+ end
1507
+ else
1508
+ table.insert(jids, jid)
1509
+ end
1510
+ end
1511
+ end
1512
+
1513
+ return jids
1514
+ end
1515
+
1516
+ function QlessQueue.deregister(...)
1517
+ redis.call('zrem', Qless.ns .. 'queues', unpack(arg))
1518
+ end
1519
+
1520
+ function QlessQueue.counts(now, name)
1521
+ if name then
1522
+ local queue = Qless.queue(name)
1523
+ local stalled = queue.locks.length(now)
1524
+ return {
1525
+ name = name,
1526
+ waiting = queue.work.length(),
1527
+ stalled = stalled,
1528
+ running = queue.locks.length() - stalled,
1529
+ scheduled = queue.scheduled.length(),
1530
+ depends = queue.depends.length(),
1531
+ recurring = queue.recurring.length(),
1532
+ paused = queue:paused()
1533
+ }
1534
+ else
1535
+ local queues = redis.call('zrange', 'ql:queues', 0, -1)
1536
+ local response = {}
1537
+ for index, qname in ipairs(queues) do
1538
+ table.insert(response, QlessQueue.counts(now, qname))
1539
+ end
1540
+ return response
1541
+ end
1542
+ end
1543
+ function QlessRecurringJob:data()
1544
+ local job = redis.call(
1545
+ 'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue',
1546
+ 'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog')
1547
+
1548
+ if not job[1] then
1549
+ return nil
1550
+ end
1551
+
1552
+ return {
1553
+ jid = job[1],
1554
+ klass = job[2],
1555
+ state = job[3],
1556
+ queue = job[4],
1557
+ priority = tonumber(job[5]),
1558
+ interval = tonumber(job[6]),
1559
+ retries = tonumber(job[7]),
1560
+ count = tonumber(job[8]),
1561
+ data = cjson.decode(job[9]),
1562
+ tags = cjson.decode(job[10]),
1563
+ backlog = tonumber(job[11] or 0)
1564
+ }
1565
+ end
1566
+
1567
+ function QlessRecurringJob:update(...)
1568
+ local options = {}
1569
+ if redis.call('exists', 'ql:r:' .. self.jid) ~= 0 then
1570
+ for i = 1, #arg, 2 do
1571
+ local key = arg[i]
1572
+ local value = arg[i+1]
1573
+ if key == 'priority' or key == 'interval' or key == 'retries' then
1574
+ value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value))
1575
+ if key == 'interval' then
1576
+ local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. self.jid, 'queue', 'interval'))
1577
+ Qless.queue(queue).recurring.update(
1578
+ value - tonumber(interval), self.jid)
1579
+ end
1580
+ redis.call('hset', 'ql:r:' .. self.jid, key, value)
1581
+ elseif key == 'data' then
1582
+ value = assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value))
1583
+ redis.call('hset', 'ql:r:' .. self.jid, 'data', cjson.encode(value))
1584
+ elseif key == 'klass' then
1585
+ redis.call('hset', 'ql:r:' .. self.jid, 'klass', value)
1586
+ elseif key == 'queue' then
1587
+ local queue_obj = Qless.queue(
1588
+ redis.call('hget', 'ql:r:' .. self.jid, 'queue'))
1589
+ local score = queue_obj.recurring.score(self.jid)
1590
+ queue_obj.recurring.remove(self.jid)
1591
+ Qless.queue(value).recurring.add(score, self.jid)
1592
+ redis.call('hset', 'ql:r:' .. self.jid, 'queue', value)
1593
+ elseif key == 'backlog' then
1594
+ value = assert(tonumber(value),
1595
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(value))
1596
+ redis.call('hset', 'ql:r:' .. self.jid, 'backlog', value)
1597
+ else
1598
+ error('Recur(): Unrecognized option "' .. key .. '"')
1599
+ end
1600
+ end
1601
+ return true
1602
+ else
1603
+ error('Recur(): No recurring job ' .. self.jid)
1604
+ end
1605
+ end
1606
+
1607
+ function QlessRecurringJob:tag(...)
1608
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
1609
+ if tags then
1610
+ tags = cjson.decode(tags)
1611
+ local _tags = {}
1612
+ for i,v in ipairs(tags) do _tags[v] = true end
1613
+
1614
+ for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end
1615
+
1616
+ tags = cjson.encode(tags)
1617
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
1618
+ return tags
1619
+ else
1620
+ return false
1621
+ end
1622
+ end
1623
+
1624
+ function QlessRecurringJob:untag(...)
1625
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
1626
+ if tags then
1627
+ tags = cjson.decode(tags)
1628
+ local _tags = {}
1629
+ for i,v in ipairs(tags) do _tags[v] = true end
1630
+ for i = 1,#arg do _tags[arg[i]] = nil end
1631
+ local results = {}
1632
+ for i, tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
1633
+ tags = cjson.encode(results)
1634
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
1635
+ return tags
1636
+ else
1637
+ return false
1638
+ end
1639
+ end
1640
+
1641
+ function QlessRecurringJob:unrecur()
1642
+ local queue = redis.call('hget', 'ql:r:' .. self.jid, 'queue')
1643
+ if queue then
1644
+ Qless.queue(queue).recurring.remove(self.jid)
1645
+ redis.call('del', 'ql:r:' .. self.jid)
1646
+ return true
1647
+ else
1648
+ return true
1649
+ end
1650
+ end
1651
+ function QlessWorker.deregister(...)
1652
+ redis.call('zrem', 'ql:workers', unpack(arg))
1653
+ end
1654
+
1655
+ function QlessWorker.counts(now, worker)
1656
+ local interval = tonumber(Qless.config.get('max-worker-age', 86400))
1657
+
1658
+ local workers = redis.call('zrangebyscore', 'ql:workers', 0, now - interval)
1659
+ for index, worker in ipairs(workers) do
1660
+ redis.call('del', 'ql:w:' .. worker .. ':jobs')
1661
+ end
1662
+
1663
+ redis.call('zremrangebyscore', 'ql:workers', 0, now - interval)
1664
+
1665
+ if worker then
1666
+ return {
1667
+ jobs = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now),
1668
+ stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0)
1669
+ }
1670
+ else
1671
+ local response = {}
1672
+ local workers = redis.call('zrevrange', 'ql:workers', 0, -1)
1673
+ for index, worker in ipairs(workers) do
1674
+ table.insert(response, {
1675
+ name = worker,
1676
+ jobs = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000),
1677
+ stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now)
1678
+ })
1679
+ end
1680
+ return response
1681
+ end
1682
+ end
1683
+ local QlessAPI = {}
1684
+
1685
+ function QlessAPI.get(now, jid)
1686
+ local data = Qless.job(jid):data()
1687
+ if not data then
1688
+ return nil
1689
+ end
1690
+ return cjson.encode(data)
1691
+ end
1692
+
1693
+ function QlessAPI.multiget(now, ...)
1694
+ local results = {}
1695
+ for i, jid in ipairs(arg) do
1696
+ table.insert(results, Qless.job(jid):data())
1697
+ end
1698
+ return cjson.encode(results)
1699
+ end
1700
+
1701
+ QlessAPI['config.get'] = function(now, key)
1702
+ if not key then
1703
+ return cjson.encode(Qless.config.get(key))
1704
+ else
1705
+ return Qless.config.get(key)
1706
+ end
1707
+ end
1708
+
1709
+ QlessAPI['config.set'] = function(now, key, value)
1710
+ return Qless.config.set(key, value)
1711
+ end
1712
+
1713
+ QlessAPI['config.unset'] = function(now, key)
1714
+ return Qless.config.unset(key)
1715
+ end
1716
+
1717
+ QlessAPI.queues = function(now, queue)
1718
+ return cjson.encode(QlessQueue.counts(now, queue))
1719
+ end
1720
+
1721
+ QlessAPI.complete = function(now, jid, worker, queue, data, ...)
1722
+ return Qless.job(jid):complete(now, worker, queue, data, unpack(arg))
1723
+ end
1724
+
1725
+ QlessAPI.failed = function(now, group, start, limit)
1726
+ return cjson.encode(Qless.failed(group, start, limit))
1727
+ end
1728
+
1729
+ QlessAPI.fail = function(now, jid, worker, group, message, data)
1730
+ return Qless.job(jid):fail(now, worker, group, message, data)
1731
+ end
1732
+
1733
+ QlessAPI.jobs = function(now, state, ...)
1734
+ return Qless.jobs(now, state, unpack(arg))
1735
+ end
1736
+
1737
+ QlessAPI.retry = function(now, jid, queue, worker, delay, group, message)
1738
+ return Qless.job(jid):retry(now, queue, worker, delay, group, message)
1739
+ end
1740
+
1741
+ QlessAPI.depends = function(now, jid, command, ...)
1742
+ return Qless.job(jid):depends(now, command, unpack(arg))
1743
+ end
1744
+
1745
+ QlessAPI.heartbeat = function(now, jid, worker, data)
1746
+ return Qless.job(jid):heartbeat(now, worker, data)
1747
+ end
1748
+
1749
+ QlessAPI.workers = function(now, worker)
1750
+ return cjson.encode(QlessWorker.counts(now, worker))
1751
+ end
1752
+
1753
+ QlessAPI.track = function(now, command, jid)
1754
+ return cjson.encode(Qless.track(now, command, jid))
1755
+ end
1756
+
1757
+ QlessAPI.tag = function(now, command, ...)
1758
+ return cjson.encode(Qless.tag(now, command, unpack(arg)))
1759
+ end
1760
+
1761
+ QlessAPI.stats = function(now, queue, date)
1762
+ return cjson.encode(Qless.queue(queue):stats(now, date))
1763
+ end
1764
+
1765
+ QlessAPI.priority = function(now, jid, priority)
1766
+ return Qless.job(jid):priority(priority)
1767
+ end
1768
+
1769
+ QlessAPI.peek = function(now, queue, count)
1770
+ local jids = Qless.queue(queue):peek(now, count)
1771
+ local response = {}
1772
+ for i, jid in ipairs(jids) do
1773
+ table.insert(response, Qless.job(jid):data())
1774
+ end
1775
+ return cjson.encode(response)
1776
+ end
1777
+
1778
+ QlessAPI.pop = function(now, queue, worker, count)
1779
+ local jids = Qless.queue(queue):pop(now, worker, count)
1780
+ local response = {}
1781
+ for i, jid in ipairs(jids) do
1782
+ table.insert(response, Qless.job(jid):data())
1783
+ end
1784
+ return cjson.encode(response)
1785
+ end
1786
+
1787
+ QlessAPI.pause = function(now, ...)
1788
+ return QlessQueue.pause(unpack(arg))
1789
+ end
1790
+
1791
+ QlessAPI.unpause = function(now, ...)
1792
+ return QlessQueue.unpause(unpack(arg))
1793
+ end
1794
+
1795
+ QlessAPI.cancel = function(now, ...)
1796
+ return Qless.cancel(unpack(arg))
1797
+ end
1798
+
1799
+ QlessAPI.timeout = function(now, jid)
1800
+ return Qless.job(jid):timeout(now)
1801
+ end
1802
+
1803
+ QlessAPI.put = function(now, queue, jid, klass, data, delay, ...)
1804
+ return Qless.queue(queue):put(now, jid, klass, data, delay, unpack(arg))
1805
+ end
1806
+
1807
+ QlessAPI.unfail = function(now, queue, group, count)
1808
+ return Qless.queue(queue):unfail(now, group, count)
1809
+ end
1810
+
1811
+ QlessAPI.recur = function(now, queue, jid, klass, data, spec, ...)
1812
+ return Qless.queue(queue):recur(now, jid, klass, data, spec, unpack(arg))
1813
+ end
1814
+
1815
+ QlessAPI.unrecur = function(now, jid)
1816
+ return Qless.recurring(jid):unrecur()
1817
+ end
1818
+
1819
+ QlessAPI['recur.get'] = function(now, jid)
1820
+ local data = Qless.recurring(jid):data()
1821
+ if not data then
1822
+ return nil
1823
+ end
1824
+ return cjson.encode(data)
1825
+ end
1826
+
1827
+ QlessAPI['recur.update'] = function(now, jid, ...)
1828
+ return Qless.recurring(jid):update(unpack(arg))
1829
+ end
1830
+
1831
+ QlessAPI['recur.tag'] = function(now, jid, ...)
1832
+ return Qless.recurring(jid):tag(unpack(arg))
1833
+ end
1834
+
1835
+ QlessAPI['recur.untag'] = function(now, jid, ...)
1836
+ return Qless.recurring(jid):untag(unpack(arg))
1837
+ end
1838
+
1839
+ QlessAPI.length = function(now, queue)
1840
+ return Qless.queue(queue):length()
1841
+ end
1842
+
1843
+ QlessAPI['worker.deregister'] = function(now, ...)
1844
+ return QlessWorker.deregister(unpack(arg))
1845
+ end
1846
+
1847
+ QlessAPI['queue.forget'] = function(now, ...)
1848
+ QlessQueue.deregister(unpack(arg))
1849
+ end
1850
+
1851
+
1852
+ if #KEYS > 0 then erorr('No Keys should be provided') end
1853
+
1854
+ local command_name = assert(table.remove(ARGV, 1), 'Must provide a command')
1855
+ local command = assert(
1856
+ QlessAPI[command_name], 'Unknown command ' .. command_name)
1857
+
1858
+ local now = tonumber(table.remove(ARGV, 1))
1859
+ local now = assert(
1860
+ now, 'Arg "now" missing or not a number: ' .. (now or 'nil'))
1861
+
1862
+ return command(now, unpack(ARGV))