reqless 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +8 -0
  3. data/README.md +648 -0
  4. data/Rakefile +117 -0
  5. data/bin/docker-build-and-test +22 -0
  6. data/exe/reqless-web +11 -0
  7. data/lib/reqless/config.rb +31 -0
  8. data/lib/reqless/failure_formatter.rb +43 -0
  9. data/lib/reqless/job.rb +496 -0
  10. data/lib/reqless/job_reservers/ordered.rb +29 -0
  11. data/lib/reqless/job_reservers/round_robin.rb +46 -0
  12. data/lib/reqless/job_reservers/shuffled_round_robin.rb +21 -0
  13. data/lib/reqless/lua/reqless-lib.lua +2965 -0
  14. data/lib/reqless/lua/reqless.lua +2545 -0
  15. data/lib/reqless/lua_script.rb +90 -0
  16. data/lib/reqless/middleware/requeue_exceptions.rb +94 -0
  17. data/lib/reqless/middleware/retry_exceptions.rb +72 -0
  18. data/lib/reqless/middleware/sentry.rb +66 -0
  19. data/lib/reqless/middleware/timeout.rb +63 -0
  20. data/lib/reqless/queue.rb +189 -0
  21. data/lib/reqless/queue_priority_pattern.rb +16 -0
  22. data/lib/reqless/server/static/css/bootstrap-responsive.css +686 -0
  23. data/lib/reqless/server/static/css/bootstrap-responsive.min.css +12 -0
  24. data/lib/reqless/server/static/css/bootstrap.css +3991 -0
  25. data/lib/reqless/server/static/css/bootstrap.min.css +689 -0
  26. data/lib/reqless/server/static/css/codemirror.css +112 -0
  27. data/lib/reqless/server/static/css/docs.css +839 -0
  28. data/lib/reqless/server/static/css/jquery.noty.css +105 -0
  29. data/lib/reqless/server/static/css/noty_theme_twitter.css +137 -0
  30. data/lib/reqless/server/static/css/style.css +200 -0
  31. data/lib/reqless/server/static/favicon.ico +0 -0
  32. data/lib/reqless/server/static/img/glyphicons-halflings-white.png +0 -0
  33. data/lib/reqless/server/static/img/glyphicons-halflings.png +0 -0
  34. data/lib/reqless/server/static/js/bootstrap-alert.js +94 -0
  35. data/lib/reqless/server/static/js/bootstrap-scrollspy.js +125 -0
  36. data/lib/reqless/server/static/js/bootstrap-tab.js +130 -0
  37. data/lib/reqless/server/static/js/bootstrap-tooltip.js +270 -0
  38. data/lib/reqless/server/static/js/bootstrap-typeahead.js +285 -0
  39. data/lib/reqless/server/static/js/bootstrap.js +1726 -0
  40. data/lib/reqless/server/static/js/bootstrap.min.js +6 -0
  41. data/lib/reqless/server/static/js/codemirror.js +2972 -0
  42. data/lib/reqless/server/static/js/jquery.noty.js +220 -0
  43. data/lib/reqless/server/static/js/mode/javascript.js +360 -0
  44. data/lib/reqless/server/static/js/theme/cobalt.css +18 -0
  45. data/lib/reqless/server/static/js/theme/eclipse.css +25 -0
  46. data/lib/reqless/server/static/js/theme/elegant.css +10 -0
  47. data/lib/reqless/server/static/js/theme/lesser-dark.css +45 -0
  48. data/lib/reqless/server/static/js/theme/monokai.css +28 -0
  49. data/lib/reqless/server/static/js/theme/neat.css +9 -0
  50. data/lib/reqless/server/static/js/theme/night.css +21 -0
  51. data/lib/reqless/server/static/js/theme/rubyblue.css +21 -0
  52. data/lib/reqless/server/static/js/theme/xq-dark.css +46 -0
  53. data/lib/reqless/server/views/_job.erb +259 -0
  54. data/lib/reqless/server/views/_job_list.erb +8 -0
  55. data/lib/reqless/server/views/_pagination.erb +7 -0
  56. data/lib/reqless/server/views/about.erb +130 -0
  57. data/lib/reqless/server/views/completed.erb +11 -0
  58. data/lib/reqless/server/views/config.erb +14 -0
  59. data/lib/reqless/server/views/failed.erb +48 -0
  60. data/lib/reqless/server/views/failed_type.erb +18 -0
  61. data/lib/reqless/server/views/job.erb +17 -0
  62. data/lib/reqless/server/views/layout.erb +451 -0
  63. data/lib/reqless/server/views/overview.erb +137 -0
  64. data/lib/reqless/server/views/queue.erb +125 -0
  65. data/lib/reqless/server/views/queues.erb +45 -0
  66. data/lib/reqless/server/views/tag.erb +6 -0
  67. data/lib/reqless/server/views/throttles.erb +38 -0
  68. data/lib/reqless/server/views/track.erb +75 -0
  69. data/lib/reqless/server/views/worker.erb +34 -0
  70. data/lib/reqless/server/views/workers.erb +14 -0
  71. data/lib/reqless/server.rb +549 -0
  72. data/lib/reqless/subscriber.rb +74 -0
  73. data/lib/reqless/test_helpers/worker_helpers.rb +55 -0
  74. data/lib/reqless/throttle.rb +57 -0
  75. data/lib/reqless/version.rb +5 -0
  76. data/lib/reqless/worker/base.rb +237 -0
  77. data/lib/reqless/worker/forking.rb +215 -0
  78. data/lib/reqless/worker/serial.rb +41 -0
  79. data/lib/reqless/worker.rb +5 -0
  80. data/lib/reqless.rb +309 -0
  81. metadata +399 -0
@@ -0,0 +1,2545 @@
1
+ -- Current SHA: 8b6600adb988e7f4922f606798b6ad64c06a245d
2
+ -- This is a generated file
3
+ local function cjsonArrayDegenerationWorkaround(array)
4
+ if #array == 0 then
5
+ return "[]"
6
+ end
7
+ return cjson.encode(array)
8
+ end
9
+ local Reqless = {
10
+ ns = 'ql:'
11
+ }
12
+
13
+ local ReqlessQueue = {
14
+ ns = Reqless.ns .. 'q:'
15
+ }
16
+ ReqlessQueue.__index = ReqlessQueue
17
+
18
+ local ReqlessWorker = {
19
+ ns = Reqless.ns .. 'w:'
20
+ }
21
+ ReqlessWorker.__index = ReqlessWorker
22
+
23
+ local ReqlessJob = {
24
+ ns = Reqless.ns .. 'j:'
25
+ }
26
+ ReqlessJob.__index = ReqlessJob
27
+
28
+ local ReqlessThrottle = {
29
+ ns = Reqless.ns .. 'th:'
30
+ }
31
+ ReqlessThrottle.__index = ReqlessThrottle
32
+
33
+ local ReqlessRecurringJob = {}
34
+ ReqlessRecurringJob.__index = ReqlessRecurringJob
35
+
36
+ Reqless.config = {}
37
+
38
+ local function table_extend(self, other)
39
+ for _, v in ipairs(other) do
40
+ table.insert(self, v)
41
+ end
42
+ end
43
+
44
+ function Reqless.publish(channel, message)
45
+ redis.call('publish', Reqless.ns .. channel, message)
46
+ end
47
+
48
+ function Reqless.job(jid)
49
+ assert(jid, 'Job(): no jid provided')
50
+ local job = {}
51
+ setmetatable(job, ReqlessJob)
52
+ job.jid = jid
53
+ return job
54
+ end
55
+
56
+ function Reqless.recurring(jid)
57
+ assert(jid, 'Recurring(): no jid provided')
58
+ local job = {}
59
+ setmetatable(job, ReqlessRecurringJob)
60
+ job.jid = jid
61
+ return job
62
+ end
63
+
64
+ function Reqless.throttle(tid)
65
+ assert(tid, 'Throttle(): no tid provided')
66
+ local throttle = ReqlessThrottle.data({id = tid})
67
+ setmetatable(throttle, ReqlessThrottle)
68
+
69
+ throttle.locks = {
70
+ length = function()
71
+ return (redis.call('zcard', ReqlessThrottle.ns .. tid .. '-locks') or 0)
72
+ end, members = function()
73
+ return redis.call('zrange', ReqlessThrottle.ns .. tid .. '-locks', 0, -1)
74
+ end, add = function(...)
75
+ if #arg > 0 then
76
+ redis.call('zadd', ReqlessThrottle.ns .. tid .. '-locks', unpack(arg))
77
+ end
78
+ end, remove = function(...)
79
+ if #arg > 0 then
80
+ return redis.call('zrem', ReqlessThrottle.ns .. tid .. '-locks', unpack(arg))
81
+ end
82
+ end, pop = function(min, max)
83
+ return redis.call('zremrangebyrank', ReqlessThrottle.ns .. tid .. '-locks', min, max)
84
+ end, peek = function(min, max)
85
+ return redis.call('zrange', ReqlessThrottle.ns .. tid .. '-locks', min, max)
86
+ end
87
+ }
88
+
89
+ throttle.pending = {
90
+ length = function()
91
+ return (redis.call('zcard', ReqlessThrottle.ns .. tid .. '-pending') or 0)
92
+ end, members = function()
93
+ return redis.call('zrange', ReqlessThrottle.ns .. tid .. '-pending', 0, -1)
94
+ end, add = function(now, jid)
95
+ redis.call('zadd', ReqlessThrottle.ns .. tid .. '-pending', now, jid)
96
+ end, remove = function(...)
97
+ if #arg > 0 then
98
+ return redis.call('zrem', ReqlessThrottle.ns .. tid .. '-pending', unpack(arg))
99
+ end
100
+ end, pop = function(min, max)
101
+ return redis.call('zremrangebyrank', ReqlessThrottle.ns .. tid .. '-pending', min, max)
102
+ end, peek = function(min, max)
103
+ return redis.call('zrange', ReqlessThrottle.ns .. tid .. '-pending', min, max)
104
+ end
105
+ }
106
+
107
+ return throttle
108
+ end
109
+
110
+ function Reqless.failed(group, start, limit)
111
+ start = assert(tonumber(start or 0),
112
+ 'Failed(): Arg "start" is not a number: ' .. (start or 'nil'))
113
+ limit = assert(tonumber(limit or 25),
114
+ 'Failed(): Arg "limit" is not a number: ' .. (limit or 'nil'))
115
+
116
+ if group then
117
+ return {
118
+ total = redis.call('llen', 'ql:f:' .. group),
119
+ jobs = redis.call('lrange', 'ql:f:' .. group, start, start + limit - 1)
120
+ }
121
+ end
122
+
123
+ local response = {}
124
+ local groups = redis.call('smembers', 'ql:failures')
125
+ for _, group in ipairs(groups) do
126
+ response[group] = redis.call('llen', 'ql:f:' .. group)
127
+ end
128
+ return response
129
+ end
130
+
131
+ function Reqless.jobs(now, state, ...)
132
+ assert(state, 'Jobs(): Arg "state" missing')
133
+ if state == 'complete' then
134
+ local offset = assert(tonumber(arg[1] or 0),
135
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[1]))
136
+ local limit = assert(tonumber(arg[2] or 25),
137
+ 'Jobs(): Arg "limit" not a number: ' .. tostring(arg[2]))
138
+ return redis.call('zrevrange', 'ql:completed', offset,
139
+ offset + limit - 1)
140
+ end
141
+
142
+ local queue_name = assert(arg[1], 'Jobs(): Arg "queue" missing')
143
+ local offset = assert(tonumber(arg[2] or 0),
144
+ 'Jobs(): Arg "offset" not a number: ' .. tostring(arg[2]))
145
+ local limit = assert(tonumber(arg[3] or 25),
146
+ 'Jobs(): Arg "limit" not a number: ' .. tostring(arg[3]))
147
+
148
+ local queue = Reqless.queue(queue_name)
149
+ if state == 'running' then
150
+ return queue.locks.peek(now, offset, limit)
151
+ elseif state == 'stalled' then
152
+ return queue.locks.expired(now, offset, limit)
153
+ elseif state == 'throttled' then
154
+ return queue.throttled.peek(now, offset, limit)
155
+ elseif state == 'scheduled' then
156
+ queue:check_scheduled(now, queue.scheduled.length())
157
+ return queue.scheduled.peek(now, offset, limit)
158
+ elseif state == 'depends' then
159
+ return queue.depends.peek(now, offset, limit)
160
+ elseif state == 'recurring' then
161
+ return queue.recurring.peek(math.huge, offset, limit)
162
+ end
163
+
164
+ error('Jobs(): Unknown type "' .. state .. '"')
165
+ end
166
+
167
+ function Reqless.track(now, command, jid)
168
+ if command ~= nil then
169
+ assert(jid, 'Track(): Arg "jid" missing')
170
+ assert(Reqless.job(jid):exists(), 'Track(): Job does not exist')
171
+ if string.lower(command) == 'track' then
172
+ Reqless.publish('track', jid)
173
+ return redis.call('zadd', 'ql:tracked', now, jid)
174
+ elseif string.lower(command) == 'untrack' then
175
+ Reqless.publish('untrack', jid)
176
+ return redis.call('zrem', 'ql:tracked', jid)
177
+ end
178
+ error('Track(): Unknown action "' .. command .. '"')
179
+ end
180
+
181
+ local response = {
182
+ jobs = {},
183
+ expired = {},
184
+ }
185
+ local jids = redis.call('zrange', 'ql:tracked', 0, -1)
186
+ for _, jid in ipairs(jids) do
187
+ local data = Reqless.job(jid):data()
188
+ if data then
189
+ table.insert(response.jobs, data)
190
+ else
191
+ table.insert(response.expired, jid)
192
+ end
193
+ end
194
+ return response
195
+ end
196
+
197
+ function Reqless.tag(now, command, ...)
198
+ assert(command,
199
+ 'Tag(): Arg "command" must be "add", "remove", "get" or "top"')
200
+
201
+ if command == 'get' then
202
+ local tag = assert(arg[1], 'Tag(): Arg "tag" missing')
203
+ local offset = assert(tonumber(arg[2] or 0),
204
+ 'Tag(): Arg "offset" not a number: ' .. tostring(arg[2]))
205
+ local limit = assert(tonumber(arg[3] or 25),
206
+ 'Tag(): Arg "limit" not a number: ' .. tostring(arg[3]))
207
+ return {
208
+ total = redis.call('zcard', 'ql:t:' .. tag),
209
+ jobs = redis.call('zrange', 'ql:t:' .. tag, offset, offset + limit - 1)
210
+ }
211
+ elseif command == 'top' then
212
+ local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1]))
213
+ local limit = assert(tonumber(arg[2] or 25), 'Tag(): Arg "limit" not a number: ' .. tostring(arg[2]))
214
+ return redis.call('zrevrangebyscore', 'ql:tags', '+inf', 2, 'limit', offset, limit)
215
+ elseif command ~= 'add' and command ~= 'remove' then
216
+ error('Tag(): First argument must be "add", "remove", "get", or "top"')
217
+ end
218
+
219
+ local jid = assert(arg[1], 'Tag(): Arg "jid" missing')
220
+ local tags = redis.call('hget', ReqlessJob.ns .. jid, 'tags')
221
+ if not tags then
222
+ error('Tag(): Job ' .. jid .. ' does not exist')
223
+ end
224
+
225
+ tags = cjson.decode(tags)
226
+ local _tags = {}
227
+ for _, v in ipairs(tags) do _tags[v] = true end
228
+
229
+ if command == 'add' then
230
+ for i=2, #arg do
231
+ local tag = arg[i]
232
+ if _tags[tag] == nil then
233
+ _tags[tag] = true
234
+ table.insert(tags, tag)
235
+ end
236
+ Reqless.job(jid):insert_tag(now, tag)
237
+ end
238
+
239
+ redis.call('hset', ReqlessJob.ns .. jid, 'tags', cjson.encode(tags))
240
+ return tags
241
+ end
242
+
243
+ for i=2, #arg do
244
+ local tag = arg[i]
245
+ _tags[tag] = nil
246
+ Reqless.job(jid):remove_tag(tag)
247
+ end
248
+
249
+ local results = {}
250
+ for _, tag in ipairs(tags) do
251
+ if _tags[tag] then
252
+ table.insert(results, tag)
253
+ end
254
+ end
255
+
256
+ redis.call('hset', ReqlessJob.ns .. jid, 'tags', cjson.encode(results))
257
+ return results
258
+ end
259
+
260
+ function Reqless.cancel(now, ...)
261
+ local dependents = {}
262
+ for _, jid in ipairs(arg) do
263
+ dependents[jid] = redis.call(
264
+ 'smembers', ReqlessJob.ns .. jid .. '-dependents') or {}
265
+ end
266
+
267
+ for _, jid in ipairs(arg) do
268
+ for j, dep in ipairs(dependents[jid]) do
269
+ if dependents[dep] == nil then
270
+ error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep ..
271
+ ' but is not mentioned to be canceled')
272
+ end
273
+ end
274
+ end
275
+
276
+ for _, jid in ipairs(arg) do
277
+ local state, queue, failure, worker = unpack(redis.call(
278
+ 'hmget', ReqlessJob.ns .. jid, 'state', 'queue', 'failure', 'worker'))
279
+
280
+ if state ~= 'complete' then
281
+ local encoded = cjson.encode({
282
+ jid = jid,
283
+ worker = worker,
284
+ event = 'canceled',
285
+ queue = queue
286
+ })
287
+ Reqless.publish('log', encoded)
288
+
289
+ if worker and (worker ~= '') then
290
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
291
+ Reqless.publish('w:' .. worker, encoded)
292
+ end
293
+
294
+ if queue then
295
+ local queue = Reqless.queue(queue)
296
+ queue:remove_job(jid)
297
+ end
298
+
299
+ local job = Reqless.job(jid)
300
+
301
+ job:throttles_release(now)
302
+
303
+ for _, j in ipairs(redis.call(
304
+ 'smembers', ReqlessJob.ns .. jid .. '-dependencies')) do
305
+ redis.call('srem', ReqlessJob.ns .. j .. '-dependents', jid)
306
+ end
307
+
308
+ if state == 'failed' then
309
+ failure = cjson.decode(failure)
310
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
311
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
312
+ redis.call('srem', 'ql:failures', failure.group)
313
+ end
314
+ local bin = failure.when - (failure.when % 86400)
315
+ local failed = redis.call(
316
+ 'hget', 'ql:s:stats:' .. bin .. ':' .. queue, 'failed')
317
+ redis.call('hset',
318
+ 'ql:s:stats:' .. bin .. ':' .. queue, 'failed', failed - 1)
319
+ end
320
+
321
+ job:delete()
322
+
323
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
324
+ Reqless.publish('canceled', jid)
325
+ end
326
+ end
327
+ end
328
+
329
+ return arg
330
+ end
331
+
332
+ Reqless.config.defaults = {
333
+ ['application'] = 'reqless',
334
+ ['grace-period'] = '10',
335
+ ['heartbeat'] = '60',
336
+ ['jobs-history'] = '604800',
337
+ ['jobs-history-count'] = '50000',
338
+ ['max-job-history'] = '100',
339
+ ['max-pop-retry'] = '1',
340
+ ['max-worker-age'] = '86400',
341
+ }
342
+
343
+ Reqless.config.get = function(key, default)
344
+ if key then
345
+ return redis.call('hget', 'ql:config', key) or
346
+ Reqless.config.defaults[key] or default
347
+ end
348
+
349
+ local reply = redis.call('hgetall', 'ql:config')
350
+ for i = 1, #reply, 2 do
351
+ Reqless.config.defaults[reply[i]] = reply[i + 1]
352
+ end
353
+ return Reqless.config.defaults
354
+ end
355
+
356
+ Reqless.config.set = function(option, value)
357
+ assert(option, 'config.set(): Arg "option" missing')
358
+ assert(value , 'config.set(): Arg "value" missing')
359
+ Reqless.publish('log', cjson.encode({
360
+ event = 'config_set',
361
+ option = option,
362
+ value = value
363
+ }))
364
+
365
+ redis.call('hset', 'ql:config', option, value)
366
+ end
367
+
368
+ Reqless.config.unset = function(option)
369
+ assert(option, 'config.unset(): Arg "option" missing')
370
+ Reqless.publish('log', cjson.encode({
371
+ event = 'config_unset',
372
+ option = option
373
+ }))
374
+
375
+ redis.call('hdel', 'ql:config', option)
376
+ end
377
+
378
+ function ReqlessJob:data(...)
379
+ local job = redis.call(
380
+ 'hmget', ReqlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue',
381
+ 'worker', 'priority', 'expires', 'retries', 'remaining', 'data',
382
+ 'tags', 'failure', 'throttles', 'spawned_from_jid')
383
+
384
+ if not job[1] then
385
+ return nil
386
+ end
387
+
388
+ local data = {
389
+ jid = job[1],
390
+ klass = job[2],
391
+ state = job[3],
392
+ queue = job[4],
393
+ worker = job[5] or '',
394
+ tracked = redis.call('zscore', 'ql:tracked', self.jid) ~= false,
395
+ priority = tonumber(job[6]),
396
+ expires = tonumber(job[7]) or 0,
397
+ retries = tonumber(job[8]),
398
+ remaining = math.floor(tonumber(job[9])),
399
+ data = job[10],
400
+ tags = cjson.decode(job[11]),
401
+ history = self:history(),
402
+ failure = cjson.decode(job[12] or '{}'),
403
+ throttles = cjson.decode(job[13] or '[]'),
404
+ spawned_from_jid = job[14],
405
+ dependents = redis.call('smembers', ReqlessJob.ns .. self.jid .. '-dependents'),
406
+ dependencies = redis.call('smembers', ReqlessJob.ns .. self.jid .. '-dependencies'),
407
+ }
408
+
409
+ if #arg > 0 then
410
+ local response = {}
411
+ for _, key in ipairs(arg) do
412
+ table.insert(response, data[key])
413
+ end
414
+ return response
415
+ end
416
+
417
+ return data
418
+ end
419
+
420
+ function ReqlessJob:complete(now, worker, queue_name, raw_data, ...)
421
+ assert(worker, 'Complete(): Arg "worker" missing')
422
+ assert(queue_name , 'Complete(): Arg "queue_name" missing')
423
+ local data = assert(cjson.decode(raw_data),
424
+ 'Complete(): Arg "data" missing or not JSON: ' .. tostring(raw_data))
425
+
426
+ local options = {}
427
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
428
+
429
+ local next_queue_name = options['next']
430
+ local delay = assert(tonumber(options['delay'] or 0))
431
+ local depends = assert(cjson.decode(options['depends'] or '[]'),
432
+ 'Complete(): Arg "depends" not JSON: ' .. tostring(options['depends']))
433
+
434
+ if options['delay'] and next_queue_name == nil then
435
+ error('Complete(): "delay" cannot be used without a "next".')
436
+ end
437
+
438
+ if options['depends'] and next_queue_name == nil then
439
+ error('Complete(): "depends" cannot be used without a "next".')
440
+ end
441
+
442
+ local bin = now - (now % 86400)
443
+
444
+ local lastworker, state, priority, retries, current_queue = unpack(
445
+ redis.call('hmget', ReqlessJob.ns .. self.jid, 'worker', 'state',
446
+ 'priority', 'retries', 'queue'))
447
+
448
+ if lastworker == false then
449
+ error('Complete(): Job does not exist')
450
+ elseif (state ~= 'running') then
451
+ error('Complete(): Job is not currently running: ' .. state)
452
+ elseif lastworker ~= worker then
453
+ error('Complete(): Job has been handed out to another worker: ' ..
454
+ tostring(lastworker))
455
+ elseif queue_name ~= current_queue then
456
+ error('Complete(): Job running in another queue: ' ..
457
+ tostring(current_queue))
458
+ end
459
+
460
+ self:history(now, 'done')
461
+
462
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'data', raw_data)
463
+
464
+ local queue = Reqless.queue(queue_name)
465
+ queue:remove_job(self.jid)
466
+
467
+ self:throttles_release(now)
468
+
469
+ local popped_time = tonumber(
470
+ redis.call('hget', ReqlessJob.ns .. self.jid, 'time') or now)
471
+ local run_time = now - popped_time
472
+ queue:stat(now, 'run', run_time)
473
+ redis.call('hset', ReqlessJob.ns .. self.jid,
474
+ 'time', string.format("%.20f", now))
475
+
476
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
477
+
478
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
479
+ Reqless.publish('completed', self.jid)
480
+ end
481
+
482
+ if next_queue_name then
483
+ local next_queue = Reqless.queue(next_queue_name)
484
+ Reqless.publish('log', cjson.encode({
485
+ jid = self.jid,
486
+ event = 'advanced',
487
+ queue = queue_name,
488
+ to = next_queue_name,
489
+ }))
490
+
491
+ self:history(now, 'put', {queue = next_queue_name})
492
+
493
+ if redis.call('zscore', 'ql:queues', next_queue_name) == false then
494
+ redis.call('zadd', 'ql:queues', now, next_queue_name)
495
+ end
496
+
497
+ redis.call('hmset', ReqlessJob.ns .. self.jid,
498
+ 'state', 'waiting',
499
+ 'worker', '',
500
+ 'failure', '{}',
501
+ 'queue', next_queue_name,
502
+ 'expires', 0,
503
+ 'remaining', tonumber(retries))
504
+
505
+ if (delay > 0) and (#depends == 0) then
506
+ next_queue.scheduled.add(now + delay, self.jid)
507
+ return 'scheduled'
508
+ end
509
+
510
+ local count = 0
511
+ for _, j in ipairs(depends) do
512
+ local state = redis.call('hget', ReqlessJob.ns .. j, 'state')
513
+ if (state and state ~= 'complete') then
514
+ count = count + 1
515
+ redis.call(
516
+ 'sadd', ReqlessJob.ns .. j .. '-dependents',self.jid)
517
+ redis.call(
518
+ 'sadd', ReqlessJob.ns .. self.jid .. '-dependencies', j)
519
+ end
520
+ end
521
+ if count > 0 then
522
+ next_queue.depends.add(now, self.jid)
523
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'state', 'depends')
524
+ if delay > 0 then
525
+ next_queue.depends.add(now, self.jid)
526
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'scheduled', now + delay)
527
+ end
528
+ return 'depends'
529
+ end
530
+
531
+ next_queue.work.add(now, priority, self.jid)
532
+ return 'waiting'
533
+ end
534
+ Reqless.publish('log', cjson.encode({
535
+ jid = self.jid,
536
+ event = 'completed',
537
+ queue = queue_name,
538
+ }))
539
+
540
+ redis.call('hmset', ReqlessJob.ns .. self.jid,
541
+ 'state', 'complete',
542
+ 'worker', '',
543
+ 'failure', '{}',
544
+ 'queue', '',
545
+ 'expires', 0,
546
+ 'remaining', tonumber(retries))
547
+
548
+ local count = Reqless.config.get('jobs-history-count')
549
+ local time = Reqless.config.get('jobs-history')
550
+
551
+ count = tonumber(count or 50000)
552
+ time = tonumber(time or 7 * 24 * 60 * 60)
553
+
554
+ redis.call('zadd', 'ql:completed', now, self.jid)
555
+
556
+ local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
557
+ for _, jid in ipairs(jids) do
558
+ Reqless.job(jid):delete()
559
+ end
560
+
561
+ redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
562
+
563
+ jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
564
+ for _, jid in ipairs(jids) do
565
+ Reqless.job(jid):delete()
566
+ end
567
+ redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
568
+
569
+ for _, j in ipairs(redis.call(
570
+ 'smembers', ReqlessJob.ns .. self.jid .. '-dependents')) do
571
+ redis.call('srem', ReqlessJob.ns .. j .. '-dependencies', self.jid)
572
+ if redis.call(
573
+ 'scard', ReqlessJob.ns .. j .. '-dependencies') == 0 then
574
+ local other_queue_name, priority, scheduled = unpack(
575
+ redis.call('hmget', ReqlessJob.ns .. j, 'queue', 'priority', 'scheduled'))
576
+ if other_queue_name then
577
+ local other_queue = Reqless.queue(other_queue_name)
578
+ other_queue.depends.remove(j)
579
+ if scheduled then
580
+ other_queue.scheduled.add(scheduled, j)
581
+ redis.call('hset', ReqlessJob.ns .. j, 'state', 'scheduled')
582
+ redis.call('hdel', ReqlessJob.ns .. j, 'scheduled')
583
+ else
584
+ other_queue.work.add(now, priority, j)
585
+ redis.call('hset', ReqlessJob.ns .. j, 'state', 'waiting')
586
+ end
587
+ end
588
+ end
589
+ end
590
+
591
+ redis.call('del', ReqlessJob.ns .. self.jid .. '-dependents')
592
+
593
+ return 'complete'
594
+ end
595
+
596
+ function ReqlessJob:fail(now, worker, group, message, data)
597
+ local worker = assert(worker , 'Fail(): Arg "worker" missing')
598
+ local group = assert(group , 'Fail(): Arg "group" missing')
599
+ local message = assert(message , 'Fail(): Arg "message" missing')
600
+
601
+ local bin = now - (now % 86400)
602
+
603
+ if data then
604
+ data = cjson.decode(data)
605
+ end
606
+
607
+ local queue_name, state, oldworker = unpack(redis.call(
608
+ 'hmget', ReqlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
609
+
610
+ if not state then
611
+ error('Fail(): Job does not exist')
612
+ elseif state ~= 'running' then
613
+ error('Fail(): Job not currently running: ' .. state)
614
+ elseif worker ~= oldworker then
615
+ error('Fail(): Job running with another worker: ' .. oldworker)
616
+ end
617
+
618
+ Reqless.publish('log', cjson.encode({
619
+ jid = self.jid,
620
+ event = 'failed',
621
+ worker = worker,
622
+ group = group,
623
+ message = message,
624
+ }))
625
+
626
+ if redis.call('zscore', 'ql:tracked', self.jid) ~= false then
627
+ Reqless.publish('failed', self.jid)
628
+ end
629
+
630
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
631
+
632
+ self:history(now, 'failed', {worker = worker, group = group})
633
+
634
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue_name, 'failures', 1)
635
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue_name, 'failed' , 1)
636
+
637
+ local queue = Reqless.queue(queue_name)
638
+ queue:remove_job(self.jid)
639
+
640
+ if data then
641
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'data', cjson.encode(data))
642
+ end
643
+
644
+ redis.call('hmset', ReqlessJob.ns .. self.jid,
645
+ 'state', 'failed',
646
+ 'worker', '',
647
+ 'expires', '',
648
+ 'failure', cjson.encode({
649
+ group = group,
650
+ message = message,
651
+ when = math.floor(now),
652
+ worker = worker
653
+ }))
654
+
655
+ self:throttles_release(now)
656
+
657
+ redis.call('sadd', 'ql:failures', group)
658
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
659
+
660
+
661
+ return self.jid
662
+ end
663
+
664
+ function ReqlessJob:retry(now, queue_name, worker, delay, group, message)
665
+ assert(queue_name , 'Retry(): Arg "queue_name" missing')
666
+ assert(worker, 'Retry(): Arg "worker" missing')
667
+ delay = assert(tonumber(delay or 0),
668
+ 'Retry(): Arg "delay" not a number: ' .. tostring(delay))
669
+
670
+ local old_queue_name, state, retries, oldworker, priority, failure = unpack(
671
+ redis.call('hmget', ReqlessJob.ns .. self.jid, 'queue', 'state',
672
+ 'retries', 'worker', 'priority', 'failure'))
673
+
674
+ if oldworker == false then
675
+ error('Retry(): Job does not exist')
676
+ elseif state ~= 'running' then
677
+ error('Retry(): Job is not currently running: ' .. state)
678
+ elseif oldworker ~= worker then
679
+ error('Retry(): Job has been given to another worker: ' .. oldworker)
680
+ end
681
+
682
+ local remaining = tonumber(redis.call(
683
+ 'hincrby', ReqlessJob.ns .. self.jid, 'remaining', -1))
684
+ redis.call('hdel', ReqlessJob.ns .. self.jid, 'grace')
685
+
686
+ Reqless.queue(old_queue_name).locks.remove(self.jid)
687
+
688
+ self:throttles_release(now)
689
+
690
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
691
+
692
+ if remaining < 0 then
693
+ local group = group or 'failed-retries-' .. queue_name
694
+ self:history(now, 'failed-retries', {group = group})
695
+
696
+ redis.call('hmset', ReqlessJob.ns .. self.jid, 'state', 'failed',
697
+ 'worker', '',
698
+ 'expires', '')
699
+ if group ~= nil and message ~= nil then
700
+ redis.call('hset', ReqlessJob.ns .. self.jid,
701
+ 'failure', cjson.encode({
702
+ group = group,
703
+ message = message,
704
+ when = math.floor(now),
705
+ worker = worker
706
+ })
707
+ )
708
+ else
709
+ redis.call('hset', ReqlessJob.ns .. self.jid,
710
+ 'failure', cjson.encode({
711
+ group = group,
712
+ message = 'Job exhausted retries in queue "' .. old_queue_name .. '"',
713
+ when = now,
714
+ worker = unpack(self:data('worker'))
715
+ }))
716
+ end
717
+
718
+ redis.call('sadd', 'ql:failures', group)
719
+ redis.call('lpush', 'ql:f:' .. group, self.jid)
720
+ local bin = now - (now % 86400)
721
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue_name, 'failures', 1)
722
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. queue_name, 'failed' , 1)
723
+ else
724
+ local queue = Reqless.queue(queue_name)
725
+ if delay > 0 then
726
+ queue.scheduled.add(now + delay, self.jid)
727
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'state', 'scheduled')
728
+ else
729
+ queue.work.add(now, priority, self.jid)
730
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'state', 'waiting')
731
+ end
732
+
733
+ if group ~= nil and message ~= nil then
734
+ redis.call('hset', ReqlessJob.ns .. self.jid,
735
+ 'failure', cjson.encode({
736
+ group = group,
737
+ message = message,
738
+ when = math.floor(now),
739
+ worker = worker
740
+ })
741
+ )
742
+ end
743
+ end
744
+
745
+ return math.floor(remaining)
746
+ end
747
+
748
+ function ReqlessJob:depends(now, command, ...)
749
+ assert(command, 'Depends(): Arg "command" missing')
750
+ if command ~= 'on' and command ~= 'off' then
751
+ error('Depends(): Argument "command" must be "on" or "off"')
752
+ end
753
+
754
+ local state = redis.call('hget', ReqlessJob.ns .. self.jid, 'state')
755
+ if state ~= 'depends' then
756
+ error('Depends(): Job ' .. self.jid ..
757
+ ' not in the depends state: ' .. tostring(state))
758
+ end
759
+
760
+ if command == 'on' then
761
+ for _, j in ipairs(arg) do
762
+ local state = redis.call('hget', ReqlessJob.ns .. j, 'state')
763
+ if (state and state ~= 'complete') then
764
+ redis.call(
765
+ 'sadd', ReqlessJob.ns .. j .. '-dependents' , self.jid)
766
+ redis.call(
767
+ 'sadd', ReqlessJob.ns .. self.jid .. '-dependencies', j)
768
+ end
769
+ end
770
+ return true
771
+ end
772
+
773
+ if arg[1] == 'all' then
774
+ for _, j in ipairs(redis.call(
775
+ 'smembers', ReqlessJob.ns .. self.jid .. '-dependencies')) do
776
+ redis.call('srem', ReqlessJob.ns .. j .. '-dependents', self.jid)
777
+ end
778
+ redis.call('del', ReqlessJob.ns .. self.jid .. '-dependencies')
779
+ local queue_name, priority = unpack(redis.call(
780
+ 'hmget', ReqlessJob.ns .. self.jid, 'queue', 'priority'))
781
+ if queue_name then
782
+ local queue = Reqless.queue(queue_name)
783
+ queue.depends.remove(self.jid)
784
+ queue.work.add(now, priority, self.jid)
785
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'state', 'waiting')
786
+ end
787
+ else
788
+ for _, j in ipairs(arg) do
789
+ redis.call('srem', ReqlessJob.ns .. j .. '-dependents', self.jid)
790
+ redis.call(
791
+ 'srem', ReqlessJob.ns .. self.jid .. '-dependencies', j)
792
+ if redis.call('scard',
793
+ ReqlessJob.ns .. self.jid .. '-dependencies') == 0 then
794
+ local queue_name, priority = unpack(redis.call(
795
+ 'hmget', ReqlessJob.ns .. self.jid, 'queue', 'priority'))
796
+ if queue_name then
797
+ local queue = Reqless.queue(queue_name)
798
+ queue.depends.remove(self.jid)
799
+ queue.work.add(now, priority, self.jid)
800
+ redis.call('hset',
801
+ ReqlessJob.ns .. self.jid, 'state', 'waiting')
802
+ end
803
+ end
804
+ end
805
+ end
806
+ return true
807
+ end
808
+
809
+ function ReqlessJob:heartbeat(now, worker, data)
810
+ assert(worker, 'Heatbeat(): Arg "worker" missing')
811
+
812
+ local queue_name = redis.call('hget', ReqlessJob.ns .. self.jid, 'queue') or ''
813
+ local expires = now + tonumber(
814
+ Reqless.config.get(queue_name .. '-heartbeat') or
815
+ Reqless.config.get('heartbeat', 60))
816
+
817
+ if data then
818
+ data = cjson.decode(data)
819
+ end
820
+
821
+ local job_worker, state = unpack(
822
+ redis.call('hmget', ReqlessJob.ns .. self.jid, 'worker', 'state'))
823
+ if job_worker == false then
824
+ error('Heartbeat(): Job does not exist')
825
+ elseif state ~= 'running' then
826
+ error('Heartbeat(): Job not currently running: ' .. state)
827
+ elseif job_worker ~= worker or #job_worker == 0 then
828
+ error('Heartbeat(): Job given out to another worker: ' .. job_worker)
829
+ end
830
+
831
+ if data then
832
+ redis.call('hmset', ReqlessJob.ns .. self.jid, 'expires',
833
+ expires, 'worker', worker, 'data', cjson.encode(data))
834
+ else
835
+ redis.call('hmset', ReqlessJob.ns .. self.jid,
836
+ 'expires', expires, 'worker', worker)
837
+ end
838
+
839
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
840
+
841
+ local queue = Reqless.queue(
842
+ redis.call('hget', ReqlessJob.ns .. self.jid, 'queue'))
843
+ queue.locks.add(expires, self.jid)
844
+ return expires
845
+ end
846
+
847
+ function ReqlessJob:priority(priority)
848
+ priority = assert(tonumber(priority),
849
+ 'Priority(): Arg "priority" missing or not a number: ' ..
850
+ tostring(priority))
851
+
852
+ local queue_name = redis.call('hget', ReqlessJob.ns .. self.jid, 'queue')
853
+
854
+ if queue_name == nil then
855
+ error('Priority(): Job ' .. self.jid .. ' does not exist')
856
+ end
857
+
858
+ if queue_name ~= '' then
859
+ local queue = Reqless.queue(queue_name)
860
+ if queue.work.score(self.jid) then
861
+ queue.work.add(0, priority, self.jid)
862
+ end
863
+ end
864
+
865
+ redis.call('hset', ReqlessJob.ns .. self.jid, 'priority', priority)
866
+ return priority
867
+ end
868
+
869
+ function ReqlessJob:update(data)
870
+ local tmp = {}
871
+ for k, v in pairs(data) do
872
+ table.insert(tmp, k)
873
+ table.insert(tmp, v)
874
+ end
875
+ redis.call('hmset', ReqlessJob.ns .. self.jid, unpack(tmp))
876
+ end
877
+
878
+ function ReqlessJob:timeout(now)
879
+ local queue_name, state, worker = unpack(redis.call('hmget',
880
+ ReqlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
881
+ if queue_name == nil then
882
+ error('Timeout(): Job does not exist')
883
+ elseif state ~= 'running' then
884
+ error('Timeout(): Job ' .. self.jid .. ' not running')
885
+ end
886
+ self:history(now, 'timed-out')
887
+ local queue = Reqless.queue(queue_name)
888
+ queue.locks.remove(self.jid)
889
+
890
+ self:throttles_release(now)
891
+
892
+ queue.work.add(now, math.huge, self.jid)
893
+ redis.call('hmset', ReqlessJob.ns .. self.jid,
894
+ 'state', 'stalled', 'expires', 0, 'worker', '')
895
+ local encoded = cjson.encode({
896
+ jid = self.jid,
897
+ event = 'lock_lost',
898
+ worker = worker,
899
+ })
900
+ Reqless.publish('w:' .. worker, encoded)
901
+ Reqless.publish('log', encoded)
902
+ return queue_name
903
+ end
904
+
905
+ function ReqlessJob:exists()
906
+ return redis.call('exists', ReqlessJob.ns .. self.jid) == 1
907
+ end
908
+
909
+ function ReqlessJob:history(now, what, item)
910
+ local history = redis.call('hget', ReqlessJob.ns .. self.jid, 'history')
911
+ if history then
912
+ history = cjson.decode(history)
913
+ for _, value in ipairs(history) do
914
+ redis.call('rpush', ReqlessJob.ns .. self.jid .. '-history',
915
+ cjson.encode({math.floor(value.put), 'put', {queue = value.queue}}))
916
+
917
+ if value.popped then
918
+ redis.call('rpush', ReqlessJob.ns .. self.jid .. '-history',
919
+ cjson.encode({math.floor(value.popped), 'popped',
920
+ {worker = value.worker}}))
921
+ end
922
+
923
+ if value.failed then
924
+ redis.call('rpush', ReqlessJob.ns .. self.jid .. '-history',
925
+ cjson.encode(
926
+ {math.floor(value.failed), 'failed', nil}))
927
+ end
928
+
929
+ if value.done then
930
+ redis.call('rpush', ReqlessJob.ns .. self.jid .. '-history',
931
+ cjson.encode(
932
+ {math.floor(value.done), 'done', nil}))
933
+ end
934
+ end
935
+ redis.call('hdel', ReqlessJob.ns .. self.jid, 'history')
936
+ end
937
+
938
+ if what == nil then
939
+ local response = {}
940
+ for _, value in ipairs(redis.call('lrange',
941
+ ReqlessJob.ns .. self.jid .. '-history', 0, -1)) do
942
+ value = cjson.decode(value)
943
+ local dict = value[3] or {}
944
+ dict['when'] = value[1]
945
+ dict['what'] = value[2]
946
+ table.insert(response, dict)
947
+ end
948
+ return response
949
+ end
950
+
951
+ local count = tonumber(Reqless.config.get('max-job-history', 100))
952
+ if count > 0 then
953
+ local obj = redis.call('lpop', ReqlessJob.ns .. self.jid .. '-history')
954
+ redis.call('ltrim', ReqlessJob.ns .. self.jid .. '-history', -count + 2, -1)
955
+ if obj ~= nil and obj ~= false then
956
+ redis.call('lpush', ReqlessJob.ns .. self.jid .. '-history', obj)
957
+ end
958
+ end
959
+ return redis.call('rpush', ReqlessJob.ns .. self.jid .. '-history',
960
+ cjson.encode({math.floor(now), what, item}))
961
+ end
962
+
963
+ function ReqlessJob:throttles_release(now)
964
+ local throttles = redis.call('hget', ReqlessJob.ns .. self.jid, 'throttles')
965
+ throttles = cjson.decode(throttles or '[]')
966
+
967
+ for _, tid in ipairs(throttles) do
968
+ Reqless.throttle(tid):release(now, self.jid)
969
+ end
970
+ end
971
+
972
+ function ReqlessJob:throttles_available()
973
+ for _, tid in ipairs(self:throttles()) do
974
+ if not Reqless.throttle(tid):available() then
975
+ return false
976
+ end
977
+ end
978
+
979
+ return true
980
+ end
981
+
982
+ function ReqlessJob:throttles_acquire(now)
983
+ if not self:throttles_available() then
984
+ return false
985
+ end
986
+
987
+ for _, tid in ipairs(self:throttles()) do
988
+ Reqless.throttle(tid):acquire(self.jid)
989
+ end
990
+
991
+ return true
992
+ end
993
+
994
+ function ReqlessJob:throttle(now)
995
+ for _, tid in ipairs(self:throttles()) do
996
+ local throttle = Reqless.throttle(tid)
997
+ if not throttle:available() then
998
+ throttle:pend(now, self.jid)
999
+ return
1000
+ end
1001
+ end
1002
+ end
1003
+
1004
+ function ReqlessJob:throttles()
1005
+ if not self._throttles then
1006
+ self._throttles = cjson.decode(redis.call('hget', ReqlessJob.ns .. self.jid, 'throttles') or '[]')
1007
+ end
1008
+
1009
+ return self._throttles
1010
+ end
1011
+
1012
+ function ReqlessJob:delete()
1013
+ local tags = redis.call('hget', ReqlessJob.ns .. self.jid, 'tags') or '[]'
1014
+ tags = cjson.decode(tags)
1015
+ for _, tag in ipairs(tags) do
1016
+ self:remove_tag(tag)
1017
+ end
1018
+ redis.call('del', ReqlessJob.ns .. self.jid)
1019
+ redis.call('del', ReqlessJob.ns .. self.jid .. '-history')
1020
+ redis.call('del', ReqlessJob.ns .. self.jid .. '-dependencies')
1021
+ end
1022
+
1023
+ function ReqlessJob:insert_tag(now, tag)
1024
+ redis.call('zadd', 'ql:t:' .. tag, now, self.jid)
1025
+ redis.call('zincrby', 'ql:tags', 1, tag)
1026
+ end
1027
+
1028
+ function ReqlessJob:remove_tag(tag)
1029
+ local namespaced_tag = 'ql:t:' .. tag
1030
+
1031
+ redis.call('zrem', namespaced_tag, self.jid)
1032
+
1033
+ local remaining = redis.call('zcard', namespaced_tag)
1034
+
1035
+ if tonumber(remaining) == 0 then
1036
+ redis.call('zrem', 'ql:tags', tag)
1037
+ else
1038
+ redis.call('zincrby', 'ql:tags', -1, tag)
1039
+ end
1040
+ end
1041
+ function Reqless.queue(name)
1042
+ assert(name, 'Queue(): no queue name provided')
1043
+ local queue = {}
1044
+ setmetatable(queue, ReqlessQueue)
1045
+ queue.name = name
1046
+
1047
+ queue.work = {
1048
+ peek = function(offset, limit)
1049
+ if limit <= 0 then
1050
+ return {}
1051
+ end
1052
+ return redis.call('zrevrange', queue:prefix('work'), offset, offset + limit - 1)
1053
+ end, remove = function(...)
1054
+ if #arg > 0 then
1055
+ return redis.call('zrem', queue:prefix('work'), unpack(arg))
1056
+ end
1057
+ end, add = function(now, priority, jid)
1058
+ return redis.call('zadd',
1059
+ queue:prefix('work'), priority - (now / 10000000000), jid)
1060
+ end, score = function(jid)
1061
+ return redis.call('zscore', queue:prefix('work'), jid)
1062
+ end, length = function()
1063
+ return redis.call('zcard', queue:prefix('work'))
1064
+ end
1065
+ }
1066
+
1067
+ queue.locks = {
1068
+ expired = function(now, offset, limit)
1069
+ return redis.call('zrangebyscore',
1070
+ queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, limit)
1071
+ end, peek = function(now, offset, limit)
1072
+ return redis.call('zrangebyscore', queue:prefix('locks'),
1073
+ now, math.huge, 'LIMIT', offset, limit)
1074
+ end, add = function(expires, jid)
1075
+ redis.call('zadd', queue:prefix('locks'), expires, jid)
1076
+ end, remove = function(...)
1077
+ if #arg > 0 then
1078
+ return redis.call('zrem', queue:prefix('locks'), unpack(arg))
1079
+ end
1080
+ end, running = function(now)
1081
+ return redis.call('zcount', queue:prefix('locks'), now, math.huge)
1082
+ end, length = function(now)
1083
+ if now then
1084
+ return redis.call('zcount', queue:prefix('locks'), 0, now)
1085
+ else
1086
+ return redis.call('zcard', queue:prefix('locks'))
1087
+ end
1088
+ end
1089
+ }
1090
+
1091
+ queue.depends = {
1092
+ peek = function(now, offset, limit)
1093
+ return redis.call('zrange',
1094
+ queue:prefix('depends'), offset, offset + limit - 1)
1095
+ end, add = function(now, jid)
1096
+ redis.call('zadd', queue:prefix('depends'), now, jid)
1097
+ end, remove = function(...)
1098
+ if #arg > 0 then
1099
+ return redis.call('zrem', queue:prefix('depends'), unpack(arg))
1100
+ end
1101
+ end, length = function()
1102
+ return redis.call('zcard', queue:prefix('depends'))
1103
+ end
1104
+ }
1105
+
1106
+
1107
+ queue.throttled = {
1108
+ length = function()
1109
+ return (redis.call('zcard', queue:prefix('throttled')) or 0)
1110
+ end, peek = function(now, offset, limit)
1111
+ return redis.call('zrange', queue:prefix('throttled'), offset, offset + limit - 1)
1112
+ end, add = function(...)
1113
+ if #arg > 0 then
1114
+ redis.call('zadd', queue:prefix('throttled'), unpack(arg))
1115
+ end
1116
+ end, remove = function(...)
1117
+ if #arg > 0 then
1118
+ return redis.call('zrem', queue:prefix('throttled'), unpack(arg))
1119
+ end
1120
+ end, pop = function(min, max)
1121
+ return redis.call('zremrangebyrank', queue:prefix('throttled'), min, max)
1122
+ end
1123
+ }
1124
+
1125
+ queue.scheduled = {
1126
+ peek = function(now, offset, limit)
1127
+ return redis.call('zrange',
1128
+ queue:prefix('scheduled'), offset, offset + limit - 1)
1129
+ end, ready = function(now, offset, limit)
1130
+ return redis.call('zrangebyscore',
1131
+ queue:prefix('scheduled'), 0, now, 'LIMIT', offset, limit)
1132
+ end, add = function(when, jid)
1133
+ redis.call('zadd', queue:prefix('scheduled'), when, jid)
1134
+ end, remove = function(...)
1135
+ if #arg > 0 then
1136
+ return redis.call('zrem', queue:prefix('scheduled'), unpack(arg))
1137
+ end
1138
+ end, length = function()
1139
+ return redis.call('zcard', queue:prefix('scheduled'))
1140
+ end
1141
+ }
1142
+
1143
+ queue.recurring = {
1144
+ peek = function(now, offset, limit)
1145
+ return redis.call('zrangebyscore', queue:prefix('recur'),
1146
+ 0, now, 'LIMIT', offset, limit)
1147
+ end, ready = function(now, offset, limit)
1148
+ end, add = function(when, jid)
1149
+ redis.call('zadd', queue:prefix('recur'), when, jid)
1150
+ end, remove = function(...)
1151
+ if #arg > 0 then
1152
+ return redis.call('zrem', queue:prefix('recur'), unpack(arg))
1153
+ end
1154
+ end, update = function(increment, jid)
1155
+ redis.call('zincrby', queue:prefix('recur'), increment, jid)
1156
+ end, score = function(jid)
1157
+ return redis.call('zscore', queue:prefix('recur'), jid)
1158
+ end, length = function()
1159
+ return redis.call('zcard', queue:prefix('recur'))
1160
+ end
1161
+ }
1162
+ return queue
1163
+ end
1164
+
1165
+ function ReqlessQueue:prefix(group)
1166
+ if group then
1167
+ return ReqlessQueue.ns .. self.name .. '-' .. group
1168
+ end
1169
+
1170
+ return ReqlessQueue.ns .. self.name
1171
+ end
1172
+
1173
+ function ReqlessQueue:stats(now, date)
1174
+ date = assert(tonumber(date),
1175
+ 'Stats(): Arg "date" missing or not a number: ' .. (date or 'nil'))
1176
+
1177
+ local bin = date - (date % 86400)
1178
+
1179
+ local histokeys = {
1180
+ 's0','s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21','s22','s23','s24','s25','s26','s27','s28','s29','s30','s31','s32','s33','s34','s35','s36','s37','s38','s39','s40','s41','s42','s43','s44','s45','s46','s47','s48','s49','s50','s51','s52','s53','s54','s55','s56','s57','s58','s59',
1181
+ 'm1','m2','m3','m4','m5','m6','m7','m8','m9','m10','m11','m12','m13','m14','m15','m16','m17','m18','m19','m20','m21','m22','m23','m24','m25','m26','m27','m28','m29','m30','m31','m32','m33','m34','m35','m36','m37','m38','m39','m40','m41','m42','m43','m44','m45','m46','m47','m48','m49','m50','m51','m52','m53','m54','m55','m56','m57','m58','m59',
1182
+ 'h1','h2','h3','h4','h5','h6','h7','h8','h9','h10','h11','h12','h13','h14','h15','h16','h17','h18','h19','h20','h21','h22','h23',
1183
+ 'd1','d2','d3','d4','d5','d6'
1184
+ }
1185
+
1186
+ local mkstats = function(name, bin, queue)
1187
+ local results = {}
1188
+
1189
+ local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
1190
+ local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
1191
+
1192
+ count = tonumber(count) or 0
1193
+ mean = tonumber(mean) or 0
1194
+ vk = tonumber(vk)
1195
+
1196
+ results.count = count or 0
1197
+ results.mean = mean or 0
1198
+ results.histogram = {}
1199
+
1200
+ if not count then
1201
+ results.std = 0
1202
+ else
1203
+ if count > 1 then
1204
+ results.std = math.sqrt(vk / (count - 1))
1205
+ else
1206
+ results.std = 0
1207
+ end
1208
+ end
1209
+
1210
+ local histogram = redis.call('hmget', key, unpack(histokeys))
1211
+ for i=1, #histokeys do
1212
+ table.insert(results.histogram, tonumber(histogram[i]) or 0)
1213
+ end
1214
+ return results
1215
+ end
1216
+
1217
+ local retries, failed, failures = unpack(redis.call('hmget', 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 'failed', 'failures'))
1218
+ return {
1219
+ retries = tonumber(retries or 0),
1220
+ failed = tonumber(failed or 0),
1221
+ failures = tonumber(failures or 0),
1222
+ wait = mkstats('wait', bin, self.name),
1223
+ run = mkstats('run' , bin, self.name)
1224
+ }
1225
+ end
1226
+
1227
+ function ReqlessQueue:peek(now, offset, limit)
1228
+ offset = assert(tonumber(offset),
1229
+ 'Peek(): Arg "offset" missing or not a number: ' .. tostring(offset))
1230
+
1231
+ limit = assert(tonumber(limit),
1232
+ 'Peek(): Arg "limit" missing or not a number: ' .. tostring(limit))
1233
+
1234
+ if limit <= 0 then
1235
+ return {}
1236
+ end
1237
+
1238
+ local offset_with_limit = offset + limit
1239
+
1240
+ local jids = self.locks.expired(now, 0, offset_with_limit)
1241
+
1242
+ local remaining_capacity = offset_with_limit - #jids
1243
+
1244
+ self:check_recurring(now, remaining_capacity)
1245
+
1246
+ self:check_scheduled(now, remaining_capacity)
1247
+
1248
+ if offset > #jids then
1249
+ return self.work.peek(offset - #jids, limit)
1250
+ end
1251
+
1252
+ table_extend(jids, self.work.peek(0, remaining_capacity))
1253
+
1254
+ if #jids < offset then
1255
+ return {}
1256
+ end
1257
+
1258
+ return {unpack(jids, offset + 1, offset_with_limit)}
1259
+ end
1260
+
1261
+ function ReqlessQueue:paused()
1262
+ return redis.call('sismember', 'ql:paused_queues', self.name) == 1
1263
+ end
1264
+
1265
+ function ReqlessQueue.pause(now, ...)
1266
+ redis.call('sadd', 'ql:paused_queues', unpack(arg))
1267
+ end
1268
+
1269
+ function ReqlessQueue.unpause(...)
1270
+ redis.call('srem', 'ql:paused_queues', unpack(arg))
1271
+ end
1272
+
1273
+ function ReqlessQueue:pop(now, worker, limit)
1274
+ assert(worker, 'Pop(): Arg "worker" missing')
1275
+ limit = assert(tonumber(limit),
1276
+ 'Pop(): Arg "limit" missing or not a number: ' .. tostring(limit))
1277
+
1278
+ if self:paused() then
1279
+ return {}
1280
+ end
1281
+
1282
+ redis.call('zadd', 'ql:workers', now, worker)
1283
+
1284
+ local dead_jids = self:invalidate_locks(now, limit) or {}
1285
+ local popped = {}
1286
+
1287
+ for _, jid in ipairs(dead_jids) do
1288
+ local success = self:pop_job(now, worker, Reqless.job(jid))
1289
+ if success then
1290
+ table.insert(popped, jid)
1291
+ end
1292
+ end
1293
+
1294
+ if not Reqless.throttle(ReqlessQueue.ns .. self.name):available() then
1295
+ return popped
1296
+ end
1297
+
1298
+
1299
+ self:check_recurring(now, limit - #dead_jids)
1300
+
1301
+ self:check_scheduled(now, limit - #dead_jids)
1302
+
1303
+
1304
+ local pop_retry_limit = tonumber(
1305
+ Reqless.config.get(self.name .. '-max-pop-retry') or
1306
+ Reqless.config.get('max-pop-retry', 1)
1307
+ )
1308
+
1309
+ while #popped < limit and pop_retry_limit > 0 do
1310
+
1311
+ local jids = self.work.peek(0, limit - #popped) or {}
1312
+
1313
+ if #jids == 0 then
1314
+ break
1315
+ end
1316
+
1317
+
1318
+ for _, jid in ipairs(jids) do
1319
+ local job = Reqless.job(jid)
1320
+ if job:throttles_acquire(now) then
1321
+ local success = self:pop_job(now, worker, job)
1322
+ if success then
1323
+ table.insert(popped, jid)
1324
+ end
1325
+ else
1326
+ self:throttle(now, job)
1327
+ end
1328
+ end
1329
+
1330
+ self.work.remove(unpack(jids))
1331
+
1332
+ pop_retry_limit = pop_retry_limit - 1
1333
+ end
1334
+
1335
+ return popped
1336
+ end
1337
+
1338
+ function ReqlessQueue:throttle(now, job)
1339
+ job:throttle(now)
1340
+ self.throttled.add(now, job.jid)
1341
+ local state = unpack(job:data('state'))
1342
+ if state ~= 'throttled' then
1343
+ job:update({state = 'throttled'})
1344
+ job:history(now, 'throttled', {queue = self.name})
1345
+ end
1346
+ end
1347
+
1348
+ function ReqlessQueue:pop_job(now, worker, job)
1349
+ local state
1350
+ local jid = job.jid
1351
+ local job_state = job:data('state')
1352
+ if not job_state then
1353
+ return false
1354
+ end
1355
+
1356
+ state = unpack(job_state)
1357
+ job:history(now, 'popped', {worker = worker})
1358
+
1359
+ local expires = now + tonumber(
1360
+ Reqless.config.get(self.name .. '-heartbeat') or
1361
+ Reqless.config.get('heartbeat', 60))
1362
+
1363
+ local time = tonumber(redis.call('hget', ReqlessJob.ns .. jid, 'time') or now)
1364
+ local waiting = now - time
1365
+ self:stat(now, 'wait', waiting)
1366
+ redis.call('hset', ReqlessJob.ns .. jid,
1367
+ 'time', string.format("%.20f", now))
1368
+
1369
+ redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
1370
+
1371
+ job:update({
1372
+ worker = worker,
1373
+ expires = expires,
1374
+ state = 'running'
1375
+ })
1376
+
1377
+ self.locks.add(expires, jid)
1378
+
1379
+ local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
1380
+ if tracked then
1381
+ Reqless.publish('popped', jid)
1382
+ end
1383
+ return true
1384
+ end
1385
+
1386
+ function ReqlessQueue:stat(now, stat, val)
1387
+ local bin = now - (now % 86400)
1388
+ local key = 'ql:s:' .. stat .. ':' .. bin .. ':' .. self.name
1389
+
1390
+ local count, mean, vk = unpack(
1391
+ redis.call('hmget', key, 'total', 'mean', 'vk'))
1392
+
1393
+ count = count or 0
1394
+ if count == 0 then
1395
+ mean = val
1396
+ vk = 0
1397
+ count = 1
1398
+ else
1399
+ count = count + 1
1400
+ local oldmean = mean
1401
+ mean = mean + (val - mean) / count
1402
+ vk = vk + (val - mean) * (val - oldmean)
1403
+ end
1404
+
1405
+ val = math.floor(val)
1406
+ if val < 60 then -- seconds
1407
+ redis.call('hincrby', key, 's' .. val, 1)
1408
+ elseif val < 3600 then -- minutes
1409
+ redis.call('hincrby', key, 'm' .. math.floor(val / 60), 1)
1410
+ elseif val < 86400 then -- hours
1411
+ redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
1412
+ else -- days
1413
+ redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
1414
+ end
1415
+ redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
1416
+ end
1417
+
1418
+ function ReqlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
1419
+ assert(jid , 'Put(): Arg "jid" missing')
1420
+ assert(klass, 'Put(): Arg "klass" missing')
1421
+ local data = assert(cjson.decode(raw_data),
1422
+ 'Put(): Arg "data" missing or not JSON: ' .. tostring(raw_data))
1423
+ delay = assert(tonumber(delay),
1424
+ 'Put(): Arg "delay" not a number: ' .. tostring(delay))
1425
+
1426
+ if #arg % 2 == 1 then
1427
+ error('Odd number of additional args: ' .. tostring(arg))
1428
+ end
1429
+ local options = {}
1430
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
1431
+
1432
+ local job = Reqless.job(jid)
1433
+ local priority, tags, oldqueue, state, failure, retries, oldworker =
1434
+ unpack(redis.call('hmget', ReqlessJob.ns .. jid, 'priority', 'tags',
1435
+ 'queue', 'state', 'failure', 'retries', 'worker'))
1436
+
1437
+ if tags then
1438
+ Reqless.tag(now, 'remove', jid, unpack(cjson.decode(tags)))
1439
+ end
1440
+
1441
+ local retries = assert(tonumber(options['retries'] or retries or 5) ,
1442
+ 'Put(): Arg "retries" not a number: ' .. tostring(options['retries']))
1443
+ local tags = assert(cjson.decode(options['tags'] or tags or '[]' ),
1444
+ 'Put(): Arg "tags" not JSON' .. tostring(options['tags']))
1445
+ local priority = assert(tonumber(options['priority'] or priority or 0),
1446
+ 'Put(): Arg "priority" not a number' .. tostring(options['priority']))
1447
+ local depends = assert(cjson.decode(options['depends'] or '[]') ,
1448
+ 'Put(): Arg "depends" not JSON: ' .. tostring(options['depends']))
1449
+ local throttles = assert(cjson.decode(options['throttles'] or '[]'),
1450
+ 'Put(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
1451
+
1452
+ if #depends > 0 then
1453
+ local new = {}
1454
+ for _, d in ipairs(depends) do new[d] = 1 end
1455
+
1456
+ local original = redis.call(
1457
+ 'smembers', ReqlessJob.ns .. jid .. '-dependencies')
1458
+ for _, dep in pairs(original) do
1459
+ if new[dep] == nil then
1460
+ redis.call('srem', ReqlessJob.ns .. dep .. '-dependents' , jid)
1461
+ redis.call('srem', ReqlessJob.ns .. jid .. '-dependencies', dep)
1462
+ end
1463
+ end
1464
+ end
1465
+
1466
+ Reqless.publish('log', cjson.encode({
1467
+ jid = jid,
1468
+ event = 'put',
1469
+ queue = self.name
1470
+ }))
1471
+
1472
+ job:history(now, 'put', {queue = self.name})
1473
+
1474
+ if oldqueue then
1475
+ local queue_obj = Reqless.queue(oldqueue)
1476
+ queue_obj:remove_job(jid)
1477
+ local old_qid = ReqlessQueue.ns .. oldqueue
1478
+ for index, throttle_name in ipairs(throttles) do
1479
+ if throttle_name == old_qid then
1480
+ table.remove(throttles, index)
1481
+ end
1482
+ end
1483
+ end
1484
+
1485
+ if oldworker and oldworker ~= '' then
1486
+ redis.call('zrem', 'ql:w:' .. oldworker .. ':jobs', jid)
1487
+ if oldworker ~= worker then
1488
+ local encoded = cjson.encode({
1489
+ jid = jid,
1490
+ event = 'lock_lost',
1491
+ worker = oldworker
1492
+ })
1493
+ Reqless.publish('w:' .. oldworker, encoded)
1494
+ Reqless.publish('log', encoded)
1495
+ end
1496
+ end
1497
+
1498
+ if state == 'complete' then
1499
+ redis.call('zrem', 'ql:completed', jid)
1500
+ end
1501
+
1502
+ for _, tag in ipairs(tags) do
1503
+ Reqless.job(jid):insert_tag(now, tag)
1504
+ end
1505
+
1506
+ if state == 'failed' then
1507
+ failure = cjson.decode(failure)
1508
+ redis.call('lrem', 'ql:f:' .. failure.group, 0, jid)
1509
+ if redis.call('llen', 'ql:f:' .. failure.group) == 0 then
1510
+ redis.call('srem', 'ql:failures', failure.group)
1511
+ end
1512
+ local bin = failure.when - (failure.when % 86400)
1513
+ redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , -1)
1514
+ end
1515
+
1516
+ table.insert(throttles, ReqlessQueue.ns .. self.name)
1517
+
1518
+ data = {
1519
+ 'jid' , jid,
1520
+ 'klass' , klass,
1521
+ 'data' , raw_data,
1522
+ 'priority' , priority,
1523
+ 'tags' , cjson.encode(tags),
1524
+ 'state' , ((delay > 0) and 'scheduled') or 'waiting',
1525
+ 'worker' , '',
1526
+ 'expires' , 0,
1527
+ 'queue' , self.name,
1528
+ 'retries' , retries,
1529
+ 'remaining', retries,
1530
+ 'time' , string.format("%.20f", now),
1531
+ 'throttles', cjson.encode(throttles)
1532
+ }
1533
+
1534
+ redis.call('hmset', ReqlessJob.ns .. jid, unpack(data))
1535
+
1536
+ for _, j in ipairs(depends) do
1537
+ local state = redis.call('hget', ReqlessJob.ns .. j, 'state')
1538
+ if (state and state ~= 'complete') then
1539
+ redis.call('sadd', ReqlessJob.ns .. j .. '-dependents' , jid)
1540
+ redis.call('sadd', ReqlessJob.ns .. jid .. '-dependencies', j)
1541
+ end
1542
+ end
1543
+
1544
+ if delay > 0 then
1545
+ if redis.call('scard', ReqlessJob.ns .. jid .. '-dependencies') > 0 then
1546
+ self.depends.add(now, jid)
1547
+ redis.call('hmset', ReqlessJob.ns .. jid,
1548
+ 'state', 'depends',
1549
+ 'scheduled', now + delay)
1550
+ else
1551
+ self.scheduled.add(now + delay, jid)
1552
+ end
1553
+ else
1554
+ local job = Reqless.job(jid)
1555
+ if redis.call('scard', ReqlessJob.ns .. jid .. '-dependencies') > 0 then
1556
+ self.depends.add(now, jid)
1557
+ redis.call('hset', ReqlessJob.ns .. jid, 'state', 'depends')
1558
+ elseif not job:throttles_available() then
1559
+ self:throttle(now, job)
1560
+ else
1561
+ self.work.add(now, priority, jid)
1562
+ end
1563
+ end
1564
+
1565
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1566
+ redis.call('zadd', 'ql:queues', now, self.name)
1567
+ end
1568
+
1569
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1570
+ Reqless.publish('put', jid)
1571
+ end
1572
+
1573
+ return jid
1574
+ end
1575
+
1576
+ function ReqlessQueue:unfail(now, group, count)
1577
+ assert(group, 'Unfail(): Arg "group" missing')
1578
+ count = assert(tonumber(count or 25),
1579
+ 'Unfail(): Arg "count" not a number: ' .. tostring(count))
1580
+ assert(count > 0, 'Unfail(): Arg "count" must be greater than zero')
1581
+
1582
+ local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1)
1583
+
1584
+ local toinsert = {}
1585
+ for _, jid in ipairs(jids) do
1586
+ local job = Reqless.job(jid)
1587
+ local data = job:data()
1588
+ job:history(now, 'put', {queue = self.name})
1589
+ redis.call('hmset', ReqlessJob.ns .. data.jid,
1590
+ 'state' , 'waiting',
1591
+ 'worker' , '',
1592
+ 'expires' , 0,
1593
+ 'queue' , self.name,
1594
+ 'remaining', data.retries or 5)
1595
+ self.work.add(now, data.priority, data.jid)
1596
+ end
1597
+
1598
+ redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1)
1599
+ if (redis.call('llen', 'ql:f:' .. group) == 0) then
1600
+ redis.call('srem', 'ql:failures', group)
1601
+ end
1602
+
1603
+ return #jids
1604
+ end
1605
+
1606
+ function ReqlessQueue:recurAtInterval(now, jid, klass, raw_data, interval, offset, ...)
1607
+ assert(jid , 'Recur(): Arg "jid" missing')
1608
+ assert(klass, 'Recur(): Arg "klass" missing')
1609
+ local data = assert(cjson.decode(raw_data),
1610
+ 'Recur(): Arg "data" not JSON: ' .. tostring(raw_data))
1611
+
1612
+ local interval = assert(tonumber(interval),
1613
+ 'Recur(): Arg "interval" not a number: ' .. tostring(interval))
1614
+ local offset = assert(tonumber(offset),
1615
+ 'Recur(): Arg "offset" not a number: ' .. tostring(offset))
1616
+ if interval <= 0 then
1617
+ error('Recur(): Arg "interval" must be greater than 0')
1618
+ end
1619
+
1620
+ if #arg % 2 == 1 then
1621
+ error('Recur(): Odd number of additional args: ' .. tostring(arg))
1622
+ end
1623
+
1624
+ local options = {}
1625
+ for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
1626
+ options.tags = assert(cjson.decode(options.tags or '{}'),
1627
+ 'Recur(): Arg "tags" must be JSON string array: ' .. tostring(
1628
+ options.tags))
1629
+ options.priority = assert(tonumber(options.priority or 0),
1630
+ 'Recur(): Arg "priority" not a number: ' .. tostring(
1631
+ options.priority))
1632
+ options.retries = assert(tonumber(options.retries or 0),
1633
+ 'Recur(): Arg "retries" not a number: ' .. tostring(
1634
+ options.retries))
1635
+ options.backlog = assert(tonumber(options.backlog or 0),
1636
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(
1637
+ options.backlog))
1638
+ options.throttles = assert(cjson.decode(options['throttles'] or '{}'),
1639
+ 'Recur(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
1640
+
1641
+ local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
1642
+ count = count or 0
1643
+
1644
+ local throttles = options['throttles'] or {}
1645
+
1646
+ if old_queue then
1647
+ Reqless.queue(old_queue).recurring.remove(jid)
1648
+
1649
+ for index, throttle_name in ipairs(throttles) do
1650
+ if throttle_name == old_queue then
1651
+ table.remove(throttles, index)
1652
+ end
1653
+ end
1654
+ end
1655
+
1656
+ table.insert(throttles, ReqlessQueue.ns .. self.name)
1657
+
1658
+ redis.call('hmset', 'ql:r:' .. jid,
1659
+ 'jid' , jid,
1660
+ 'klass' , klass,
1661
+ 'data' , raw_data,
1662
+ 'priority' , options.priority,
1663
+ 'tags' , cjson.encode(options.tags or {}),
1664
+ 'state' , 'recur',
1665
+ 'queue' , self.name,
1666
+ 'type' , 'interval',
1667
+ 'count' , count,
1668
+ 'interval' , interval,
1669
+ 'retries' , options.retries,
1670
+ 'backlog' , options.backlog,
1671
+ 'throttles', cjson.encode(throttles))
1672
+ self.recurring.add(now + offset, jid)
1673
+
1674
+ if redis.call('zscore', 'ql:queues', self.name) == false then
1675
+ redis.call('zadd', 'ql:queues', now, self.name)
1676
+ end
1677
+
1678
+ return jid
1679
+ end
1680
+
1681
+ function ReqlessQueue:length()
1682
+ return self.locks.length() + self.work.length() + self.scheduled.length()
1683
+ end
1684
+
1685
+ function ReqlessQueue:remove_job(jid)
1686
+ self.work.remove(jid)
1687
+ self.locks.remove(jid)
1688
+ self.throttled.remove(jid)
1689
+ self.depends.remove(jid)
1690
+ self.scheduled.remove(jid)
1691
+ end
1692
+
1693
+ function ReqlessQueue:check_recurring(now, count)
1694
+ if count <= 0 then
1695
+ return
1696
+ end
1697
+ local moved = 0
1698
+ local r = self.recurring.peek(now, 0, count)
1699
+ for _, jid in ipairs(r) do
1700
+ local r = redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
1701
+ 'tags', 'retries', 'interval', 'backlog', 'throttles')
1702
+ local klass, data, priority, tags, retries, interval, backlog, throttles = unpack(
1703
+ redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
1704
+ 'tags', 'retries', 'interval', 'backlog', 'throttles'))
1705
+ local _tags = cjson.decode(tags)
1706
+ local score = math.floor(tonumber(self.recurring.score(jid)))
1707
+ interval = tonumber(interval)
1708
+
1709
+ backlog = tonumber(backlog or 0)
1710
+ if backlog ~= 0 then
1711
+ local num = ((now - score) / interval)
1712
+ if num > backlog then
1713
+ score = score + (
1714
+ math.ceil(num - backlog) * interval
1715
+ )
1716
+ end
1717
+ end
1718
+
1719
+ while (score <= now) and (moved < count) do
1720
+ local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
1721
+ moved = moved + 1
1722
+
1723
+ local child_jid = jid .. '-' .. count
1724
+
1725
+ for _, tag in ipairs(_tags) do
1726
+ Reqless.job(child_jid):insert_tag(now, tag)
1727
+ end
1728
+
1729
+ redis.call('hmset', ReqlessJob.ns .. child_jid,
1730
+ 'jid' , child_jid,
1731
+ 'klass' , klass,
1732
+ 'data' , data,
1733
+ 'priority' , priority,
1734
+ 'tags' , tags,
1735
+ 'state' , 'waiting',
1736
+ 'worker' , '',
1737
+ 'expires' , 0,
1738
+ 'queue' , self.name,
1739
+ 'retries' , retries,
1740
+ 'remaining', retries,
1741
+ 'time' , string.format("%.20f", score),
1742
+ 'throttles', throttles,
1743
+ 'spawned_from_jid', jid)
1744
+
1745
+ Reqless.job(child_jid):history(score, 'put', {queue = self.name})
1746
+
1747
+ self.work.add(score, priority, child_jid)
1748
+
1749
+ score = score + interval
1750
+ self.recurring.add(score, jid)
1751
+ end
1752
+ end
1753
+ end
1754
+
1755
+ function ReqlessQueue:check_scheduled(now, count)
1756
+ if count <= 0 then
1757
+ return
1758
+ end
1759
+ local scheduled = self.scheduled.ready(now, 0, count)
1760
+ for _, jid in ipairs(scheduled) do
1761
+ local priority = tonumber(
1762
+ redis.call('hget', ReqlessJob.ns .. jid, 'priority') or 0)
1763
+ self.work.add(now, priority, jid)
1764
+ self.scheduled.remove(jid)
1765
+
1766
+ redis.call('hset', ReqlessJob.ns .. jid, 'state', 'waiting')
1767
+ end
1768
+ end
1769
+
1770
+ function ReqlessQueue:invalidate_locks(now, count)
1771
+ local jids = {}
1772
+ for _, jid in ipairs(self.locks.expired(now, 0, count)) do
1773
+ local worker, failure = unpack(
1774
+ redis.call('hmget', ReqlessJob.ns .. jid, 'worker', 'failure'))
1775
+ redis.call('zrem', 'ql:w:' .. worker .. ':jobs', jid)
1776
+
1777
+ local grace_period = tonumber(Reqless.config.get('grace-period'))
1778
+
1779
+ local courtesy_sent = tonumber(
1780
+ redis.call('hget', ReqlessJob.ns .. jid, 'grace') or 0)
1781
+
1782
+ local send_message = (courtesy_sent ~= 1)
1783
+ local invalidate = not send_message
1784
+
1785
+ if grace_period <= 0 then
1786
+ send_message = true
1787
+ invalidate = true
1788
+ end
1789
+
1790
+ if send_message then
1791
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1792
+ Reqless.publish('stalled', jid)
1793
+ end
1794
+ Reqless.job(jid):history(now, 'timed-out')
1795
+ redis.call('hset', ReqlessJob.ns .. jid, 'grace', 1)
1796
+
1797
+ local encoded = cjson.encode({
1798
+ jid = jid,
1799
+ event = 'lock_lost',
1800
+ worker = worker,
1801
+ })
1802
+ Reqless.publish('w:' .. worker, encoded)
1803
+ Reqless.publish('log', encoded)
1804
+ self.locks.add(now + grace_period, jid)
1805
+
1806
+ local bin = now - (now % 86400)
1807
+ redis.call('hincrby',
1808
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'retries', 1)
1809
+ end
1810
+
1811
+ if invalidate then
1812
+ redis.call('hdel', ReqlessJob.ns .. jid, 'grace', 0)
1813
+
1814
+ local remaining = tonumber(redis.call(
1815
+ 'hincrby', ReqlessJob.ns .. jid, 'remaining', -1))
1816
+
1817
+ if remaining < 0 then
1818
+ self.work.remove(jid)
1819
+ self.locks.remove(jid)
1820
+ self.scheduled.remove(jid)
1821
+
1822
+ local job = Reqless.job(jid)
1823
+ local job_data = Reqless.job(jid):data()
1824
+ local queue = job_data['queue']
1825
+ local group = 'failed-retries-' .. queue
1826
+
1827
+ job:throttles_release(now)
1828
+
1829
+ job:history(now, 'failed', {group = group})
1830
+ redis.call('hmset', ReqlessJob.ns .. jid, 'state', 'failed',
1831
+ 'worker', '',
1832
+ 'expires', '')
1833
+ redis.call('hset', ReqlessJob.ns .. jid,
1834
+ 'failure', cjson.encode({
1835
+ group = group,
1836
+ message = 'Job exhausted retries in queue "' .. self.name .. '"',
1837
+ when = now,
1838
+ worker = unpack(job:data('worker'))
1839
+ }))
1840
+
1841
+ redis.call('sadd', 'ql:failures', group)
1842
+ redis.call('lpush', 'ql:f:' .. group, jid)
1843
+
1844
+ if redis.call('zscore', 'ql:tracked', jid) ~= false then
1845
+ Reqless.publish('failed', jid)
1846
+ end
1847
+ Reqless.publish('log', cjson.encode({
1848
+ jid = jid,
1849
+ event = 'failed',
1850
+ group = group,
1851
+ worker = worker,
1852
+ message =
1853
+ 'Job exhausted retries in queue "' .. self.name .. '"'
1854
+ }))
1855
+
1856
+ local bin = now - (now % 86400)
1857
+ redis.call('hincrby',
1858
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'failures', 1)
1859
+ redis.call('hincrby',
1860
+ 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed' , 1)
1861
+ else
1862
+ table.insert(jids, jid)
1863
+ end
1864
+ end
1865
+ end
1866
+
1867
+ return jids
1868
+ end
1869
+
1870
+ function ReqlessQueue.deregister(...)
1871
+ redis.call('zrem', Reqless.ns .. 'queues', unpack(arg))
1872
+ end
1873
+
1874
+ function ReqlessQueue.counts(now, name)
1875
+ if name then
1876
+ local queue = Reqless.queue(name)
1877
+ local stalled = queue.locks.length(now)
1878
+ queue:check_scheduled(now, queue.scheduled.length())
1879
+ return {
1880
+ name = name,
1881
+ waiting = queue.work.length(),
1882
+ stalled = stalled,
1883
+ running = queue.locks.length() - stalled,
1884
+ throttled = queue.throttled.length(),
1885
+ scheduled = queue.scheduled.length(),
1886
+ depends = queue.depends.length(),
1887
+ recurring = queue.recurring.length(),
1888
+ paused = queue:paused()
1889
+ }
1890
+ end
1891
+
1892
+ local queues = redis.call('zrange', 'ql:queues', 0, -1)
1893
+ local response = {}
1894
+ for _, qname in ipairs(queues) do
1895
+ table.insert(response, ReqlessQueue.counts(now, qname))
1896
+ end
1897
+ return response
1898
+ end
1899
+ local ReqlessQueuePatterns = {
1900
+ default_identifiers_default_pattern = '["*"]',
1901
+ ns = Reqless.ns .. "qp:",
1902
+ }
1903
+ ReqlessQueuePatterns.__index = ReqlessQueuePatterns
1904
+
1905
+ ReqlessQueuePatterns['getIdentifierPatterns'] = function(now)
1906
+ local reply = redis.call('hgetall', ReqlessQueuePatterns.ns .. 'identifiers')
1907
+
1908
+ if #reply == 0 then
1909
+ reply = redis.call('hgetall', 'qmore:dynamic')
1910
+ end
1911
+
1912
+ local identifierPatterns = {
1913
+ ['default'] = ReqlessQueuePatterns.default_identifiers_default_pattern,
1914
+ }
1915
+ for i = 1, #reply, 2 do
1916
+ identifierPatterns[reply[i]] = reply[i + 1]
1917
+ end
1918
+
1919
+ return identifierPatterns
1920
+ end
1921
+
1922
+ ReqlessQueuePatterns['setIdentifierPatterns'] = function(now, ...)
1923
+ if #arg % 2 == 1 then
1924
+ error('Odd number of identifier patterns: ' .. tostring(arg))
1925
+ end
1926
+ local key = ReqlessQueuePatterns.ns .. 'identifiers'
1927
+
1928
+ local goodDefault = false;
1929
+ local identifierPatterns = {}
1930
+ for i = 1, #arg, 2 do
1931
+ local key = arg[i]
1932
+ local serializedValues = arg[i + 1]
1933
+
1934
+ local values = cjson.decode(serializedValues)
1935
+
1936
+ if #values > 0 then
1937
+ if key == 'default' then
1938
+ goodDefault = true
1939
+ end
1940
+ table.insert(identifierPatterns, key)
1941
+ table.insert(identifierPatterns, serializedValues)
1942
+ end
1943
+ end
1944
+
1945
+ if not goodDefault then
1946
+ table.insert(identifierPatterns, "default")
1947
+ table.insert(
1948
+ identifierPatterns,
1949
+ ReqlessQueuePatterns.default_identifiers_default_pattern
1950
+ )
1951
+ end
1952
+
1953
+ redis.call('del', key, 'qmore:dynamic')
1954
+ redis.call('hset', key, unpack(identifierPatterns))
1955
+ end
1956
+
1957
+ ReqlessQueuePatterns['getPriorityPatterns'] = function(now)
1958
+ local reply = redis.call('lrange', ReqlessQueuePatterns.ns .. 'priorities', 0, -1)
1959
+
1960
+ if #reply == 0 then
1961
+ reply = redis.call('lrange', 'qmore:priority', 0, -1)
1962
+ end
1963
+
1964
+ return reply
1965
+ end
1966
+
1967
+ ReqlessQueuePatterns['setPriorityPatterns'] = function(now, ...)
1968
+ local key = ReqlessQueuePatterns.ns .. 'priorities'
1969
+ redis.call('del', key)
1970
+ redis.call('del', 'qmore:priority')
1971
+ if #arg > 0 then
1972
+ redis.call('rpush', key, unpack(arg))
1973
+ end
1974
+ end
1975
+ function ReqlessRecurringJob:data()
1976
+ local job = redis.call(
1977
+ 'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue',
1978
+ 'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog', 'throttles')
1979
+
1980
+ if not job[1] then
1981
+ return nil
1982
+ end
1983
+
1984
+ return {
1985
+ jid = job[1],
1986
+ klass = job[2],
1987
+ state = job[3],
1988
+ queue = job[4],
1989
+ priority = tonumber(job[5]),
1990
+ interval = tonumber(job[6]),
1991
+ retries = tonumber(job[7]),
1992
+ count = tonumber(job[8]),
1993
+ data = job[9],
1994
+ tags = cjson.decode(job[10]),
1995
+ backlog = tonumber(job[11] or 0),
1996
+ throttles = cjson.decode(job[12] or '[]'),
1997
+ }
1998
+ end
1999
+
2000
+ function ReqlessRecurringJob:update(now, ...)
2001
+ local options = {}
2002
+ if redis.call('exists', 'ql:r:' .. self.jid) == 0 then
2003
+ error('Recur(): No recurring job ' .. self.jid)
2004
+ end
2005
+
2006
+ for i = 1, #arg, 2 do
2007
+ local key = arg[i]
2008
+ local value = arg[i+1]
2009
+ assert(value, 'No value provided for ' .. tostring(key))
2010
+ if key == 'priority' or key == 'interval' or key == 'retries' then
2011
+ value = assert(tonumber(value), 'Recur(): Arg "' .. key .. '" must be a number: ' .. tostring(value))
2012
+ if key == 'interval' then
2013
+ local queue, interval = unpack(redis.call('hmget', 'ql:r:' .. self.jid, 'queue', 'interval'))
2014
+ Reqless.queue(queue).recurring.update(
2015
+ value - tonumber(interval), self.jid)
2016
+ end
2017
+ redis.call('hset', 'ql:r:' .. self.jid, key, value)
2018
+ elseif key == 'data' then
2019
+ assert(cjson.decode(value), 'Recur(): Arg "data" is not JSON-encoded: ' .. tostring(value))
2020
+ redis.call('hset', 'ql:r:' .. self.jid, 'data', value)
2021
+ elseif key == 'klass' then
2022
+ redis.call('hset', 'ql:r:' .. self.jid, 'klass', value)
2023
+ elseif key == 'queue' then
2024
+ local old_queue_name = redis.call('hget', 'ql:r:' .. self.jid, 'queue')
2025
+ local queue_obj = Reqless.queue(old_queue_name)
2026
+ local score = queue_obj.recurring.score(self.jid)
2027
+
2028
+ queue_obj.recurring.remove(self.jid)
2029
+ local throttles = cjson.decode(redis.call('hget', 'ql:r:' .. self.jid, 'throttles') or '{}')
2030
+ for index, throttle_name in ipairs(throttles) do
2031
+ if throttle_name == ReqlessQueue.ns .. old_queue_name then
2032
+ table.remove(throttles, index)
2033
+ end
2034
+ end
2035
+
2036
+
2037
+ table.insert(throttles, ReqlessQueue.ns .. value)
2038
+ redis.call('hset', 'ql:r:' .. self.jid, 'throttles', cjson.encode(throttles))
2039
+
2040
+ Reqless.queue(value).recurring.add(score, self.jid)
2041
+ redis.call('hset', 'ql:r:' .. self.jid, 'queue', value)
2042
+ if redis.call('zscore', 'ql:queues', value) == false then
2043
+ redis.call('zadd', 'ql:queues', now, value)
2044
+ end
2045
+ elseif key == 'backlog' then
2046
+ value = assert(tonumber(value),
2047
+ 'Recur(): Arg "backlog" not a number: ' .. tostring(value))
2048
+ redis.call('hset', 'ql:r:' .. self.jid, 'backlog', value)
2049
+ elseif key == 'throttles' then
2050
+ local throttles = assert(cjson.decode(value), 'Recur(): Arg "throttles" is not JSON-encoded: ' .. tostring(value))
2051
+ redis.call('hset', 'ql:r:' .. self.jid, 'throttles', cjson.encode(throttles))
2052
+ else
2053
+ error('Recur(): Unrecognized option "' .. key .. '"')
2054
+ end
2055
+ end
2056
+
2057
+ return true
2058
+ end
2059
+
2060
+ function ReqlessRecurringJob:tag(...)
2061
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
2062
+ if not tags then
2063
+ error('Tag(): Job ' .. self.jid .. ' does not exist')
2064
+ end
2065
+
2066
+ tags = cjson.decode(tags)
2067
+ local _tags = {}
2068
+ for _, v in ipairs(tags) do
2069
+ _tags[v] = true
2070
+ end
2071
+
2072
+ for i = 1, #arg do
2073
+ if _tags[arg[i]] == nil then
2074
+ table.insert(tags, arg[i])
2075
+ end
2076
+ end
2077
+
2078
+ tags = cjsonArrayDegenerationWorkaround(tags)
2079
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
2080
+
2081
+ return tags
2082
+ end
2083
+
2084
+ function ReqlessRecurringJob:untag(...)
2085
+ local tags = redis.call('hget', 'ql:r:' .. self.jid, 'tags')
2086
+
2087
+ if not tags then
2088
+ error('Untag(): Job ' .. self.jid .. ' does not exist')
2089
+ end
2090
+
2091
+ tags = cjson.decode(tags)
2092
+
2093
+ local _tags = {}
2094
+ for _, v in ipairs(tags) do
2095
+ _tags[v] = true
2096
+ end
2097
+
2098
+ for i = 1, #arg do
2099
+ _tags[arg[i]] = nil
2100
+ end
2101
+
2102
+ local results = {}
2103
+ for _, tag in ipairs(tags) do
2104
+ if _tags[tag] then
2105
+ table.insert(results, tag)
2106
+ end
2107
+ end
2108
+
2109
+ tags = cjson.encode(results)
2110
+ redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
2111
+
2112
+ return tags
2113
+ end
2114
+
2115
+ function ReqlessRecurringJob:cancel()
2116
+ local queue = redis.call('hget', 'ql:r:' .. self.jid, 'queue')
2117
+ if queue then
2118
+ Reqless.queue(queue).recurring.remove(self.jid)
2119
+ redis.call('del', 'ql:r:' .. self.jid)
2120
+ end
2121
+
2122
+ return true
2123
+ end
2124
+ function ReqlessWorker.deregister(...)
2125
+ redis.call('zrem', 'ql:workers', unpack(arg))
2126
+ end
2127
+
2128
+ function ReqlessWorker.counts(now, worker)
2129
+ local interval = tonumber(Reqless.config.get('max-worker-age', 86400))
2130
+
2131
+ local workers = redis.call('zrangebyscore', 'ql:workers', 0, now - interval)
2132
+ for _, worker in ipairs(workers) do
2133
+ redis.call('del', 'ql:w:' .. worker .. ':jobs')
2134
+ end
2135
+
2136
+ redis.call('zremrangebyscore', 'ql:workers', 0, now - interval)
2137
+
2138
+ if worker then
2139
+ return {
2140
+ jobs = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now + 8640000, now),
2141
+ stalled = redis.call('zrevrangebyscore', 'ql:w:' .. worker .. ':jobs', now, 0)
2142
+ }
2143
+ end
2144
+
2145
+ local response = {}
2146
+ local workers = redis.call('zrevrange', 'ql:workers', 0, -1)
2147
+ for _, worker in ipairs(workers) do
2148
+ table.insert(response, {
2149
+ name = worker,
2150
+ jobs = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', now, now + 8640000),
2151
+ stalled = redis.call('zcount', 'ql:w:' .. worker .. ':jobs', 0, now)
2152
+ })
2153
+ end
2154
+ return response
2155
+ end
2156
+ function ReqlessThrottle:data()
2157
+ local data = {
2158
+ id = self.id,
2159
+ maximum = 0
2160
+ }
2161
+
2162
+ local throttle = redis.call('hmget', ReqlessThrottle.ns .. self.id, 'id', 'maximum')
2163
+
2164
+ if throttle[2] then
2165
+ data.maximum = tonumber(throttle[2])
2166
+ end
2167
+
2168
+ return data
2169
+ end
2170
+
2171
+ function ReqlessThrottle:dataWithTtl()
2172
+ local data = self:data()
2173
+ data.ttl = self:ttl()
2174
+ return data
2175
+ end
2176
+
2177
+ function ReqlessThrottle:set(data, expiration)
2178
+ redis.call('hmset', ReqlessThrottle.ns .. self.id, 'id', self.id, 'maximum', data.maximum)
2179
+ if expiration > 0 then
2180
+ redis.call('expire', ReqlessThrottle.ns .. self.id, expiration)
2181
+ end
2182
+ end
2183
+
2184
+ function ReqlessThrottle:unset()
2185
+ redis.call('del', ReqlessThrottle.ns .. self.id)
2186
+ end
2187
+
2188
+ function ReqlessThrottle:acquire(jid)
2189
+ if not self:available() then
2190
+ return false
2191
+ end
2192
+
2193
+ self.locks.add(1, jid)
2194
+ return true
2195
+ end
2196
+
2197
+ function ReqlessThrottle:pend(now, jid)
2198
+ self.pending.add(now, jid)
2199
+ end
2200
+
2201
+ function ReqlessThrottle:release(now, jid)
2202
+ if self.locks.remove(jid) == 0 then
2203
+ self.pending.remove(jid)
2204
+ end
2205
+
2206
+ local available_locks = self:locks_available()
2207
+ if self.pending.length() == 0 or available_locks < 1 then
2208
+ return
2209
+ end
2210
+
2211
+ for _, jid in ipairs(self.pending.peek(0, available_locks - 1)) do
2212
+ local job = Reqless.job(jid)
2213
+ local data = job:data()
2214
+ local queue = Reqless.queue(data['queue'])
2215
+
2216
+ queue.throttled.remove(jid)
2217
+ queue.work.add(now, data.priority, jid)
2218
+ end
2219
+
2220
+ local popped = self.pending.pop(0, available_locks - 1)
2221
+ end
2222
+
2223
+ function ReqlessThrottle:available()
2224
+ return self.maximum == 0 or self.locks.length() < self.maximum
2225
+ end
2226
+
2227
+ function ReqlessThrottle:ttl()
2228
+ return redis.call('ttl', ReqlessThrottle.ns .. self.id)
2229
+ end
2230
+
2231
+ function ReqlessThrottle:locks_available()
2232
+ if self.maximum == 0 then
2233
+ return 10
2234
+ end
2235
+
2236
+ return self.maximum - self.locks.length()
2237
+ end
2238
+ local ReqlessAPI = {}
2239
+
2240
+ ReqlessAPI['config.get'] = function(now, key)
2241
+ assert(key, "config.get(): Argument 'key' missing")
2242
+ return Reqless.config.get(key)
2243
+ end
2244
+
2245
+ ReqlessAPI['config.getAll'] = function(now)
2246
+ return cjson.encode(Reqless.config.get(nil))
2247
+ end
2248
+
2249
+ ReqlessAPI['config.set'] = function(now, key, value)
2250
+ Reqless.config.set(key, value)
2251
+ end
2252
+
2253
+ ReqlessAPI['config.unset'] = function(now, key)
2254
+ Reqless.config.unset(key)
2255
+ end
2256
+
2257
+ ReqlessAPI['failureGroups.counts'] = function(now, start, limit)
2258
+ return cjson.encode(Reqless.failed(nil, start, limit))
2259
+ end
2260
+
2261
+ ReqlessAPI['job.addDependency'] = function(now, jid, ...)
2262
+ return Reqless.job(jid):depends(now, "on", unpack(arg))
2263
+ end
2264
+
2265
+ ReqlessAPI['job.addTag'] = function(now, jid, ...)
2266
+ local result = Reqless.tag(now, 'add', jid, unpack(arg))
2267
+ return cjsonArrayDegenerationWorkaround(result)
2268
+ end
2269
+
2270
+ ReqlessAPI['job.cancel'] = function(now, ...)
2271
+ return Reqless.cancel(now, unpack(arg))
2272
+ end
2273
+
2274
+ ReqlessAPI['job.complete'] = function(now, jid, worker, queue, data)
2275
+ return Reqless.job(jid):complete(now, worker, queue, data)
2276
+ end
2277
+
2278
+ ReqlessAPI['job.completeAndRequeue'] = function(now, jid, worker, queue, data, next_queue, ...)
2279
+ return Reqless.job(jid):complete(now, worker, queue, data, 'next', next_queue, unpack(arg))
2280
+ end
2281
+
2282
+ ReqlessAPI['job.fail'] = function(now, jid, worker, group, message, data)
2283
+ return Reqless.job(jid):fail(now, worker, group, message, data)
2284
+ end
2285
+
2286
+ ReqlessAPI['job.get'] = function(now, jid)
2287
+ local data = Reqless.job(jid):data()
2288
+ if data then
2289
+ return cjson.encode(data)
2290
+ end
2291
+ end
2292
+
2293
+ ReqlessAPI['job.getMulti'] = function(now, ...)
2294
+ local results = {}
2295
+ for _, jid in ipairs(arg) do
2296
+ table.insert(results, Reqless.job(jid):data())
2297
+ end
2298
+ return cjsonArrayDegenerationWorkaround(results)
2299
+ end
2300
+
2301
+ ReqlessAPI['job.heartbeat'] = function(now, jid, worker, data)
2302
+ return Reqless.job(jid):heartbeat(now, worker, data)
2303
+ end
2304
+
2305
+ ReqlessAPI['job.log'] = function(now, jid, message, data)
2306
+ assert(jid, "Log(): Argument 'jid' missing")
2307
+ assert(message, "Log(): Argument 'message' missing")
2308
+ if data then
2309
+ data = assert(cjson.decode(data),
2310
+ "Log(): Argument 'data' not cjson: " .. tostring(data))
2311
+ end
2312
+
2313
+ local job = Reqless.job(jid)
2314
+ assert(job:exists(), 'Log(): Job ' .. jid .. ' does not exist')
2315
+ job:history(now, message, data)
2316
+ end
2317
+
2318
+ ReqlessAPI['job.removeDependency'] = function(now, jid, ...)
2319
+ return Reqless.job(jid):depends(now, "off", unpack(arg))
2320
+ end
2321
+
2322
+ ReqlessAPI['job.removeTag'] = function(now, jid, ...)
2323
+ local result = Reqless.tag(now, 'remove', jid, unpack(arg))
2324
+ return cjsonArrayDegenerationWorkaround(result)
2325
+ end
2326
+
2327
+ ReqlessAPI['job.requeue'] = function(now, worker, queue, jid, klass, data, delay, ...)
2328
+ local job = Reqless.job(jid)
2329
+ assert(job:exists(), 'Requeue(): Job ' .. jid .. ' does not exist')
2330
+ return ReqlessAPI['queue.put'](now, worker, queue, jid, klass, data, delay, unpack(arg))
2331
+ end
2332
+
2333
+ ReqlessAPI['job.retry'] = function(now, jid, queue, worker, delay, group, message)
2334
+ return Reqless.job(jid):retry(now, queue, worker, delay, group, message)
2335
+ end
2336
+
2337
+ ReqlessAPI['job.setPriority'] = function(now, jid, priority)
2338
+ return Reqless.job(jid):priority(priority)
2339
+ end
2340
+
2341
+ ReqlessAPI['job.timeout'] = function(now, ...)
2342
+ for _, jid in ipairs(arg) do
2343
+ Reqless.job(jid):timeout(now)
2344
+ end
2345
+ end
2346
+
2347
+ ReqlessAPI['job.track'] = function(now, jid)
2348
+ return cjson.encode(Reqless.track(now, 'track', jid))
2349
+ end
2350
+
2351
+ ReqlessAPI['job.untrack'] = function(now, jid)
2352
+ return cjson.encode(Reqless.track(now, 'untrack', jid))
2353
+ end
2354
+
2355
+ ReqlessAPI["jobs.completed"] = function(now, offset, limit)
2356
+ local result = Reqless.jobs(now, 'complete', offset, limit)
2357
+ return cjsonArrayDegenerationWorkaround(result)
2358
+ end
2359
+
2360
+ ReqlessAPI['jobs.failedByGroup'] = function(now, group, start, limit)
2361
+ return cjson.encode(Reqless.failed(group, start, limit))
2362
+ end
2363
+
2364
+ ReqlessAPI['jobs.tagged'] = function(now, tag, ...)
2365
+ return cjson.encode(Reqless.tag(now, 'get', tag, unpack(arg)))
2366
+ end
2367
+
2368
+ ReqlessAPI['jobs.tracked'] = function(now)
2369
+ return cjson.encode(Reqless.track(now))
2370
+ end
2371
+
2372
+ ReqlessAPI['queue.counts'] = function(now, queue)
2373
+ return cjson.encode(ReqlessQueue.counts(now, queue))
2374
+ end
2375
+
2376
+ ReqlessAPI['queue.forget'] = function(now, ...)
2377
+ ReqlessQueue.deregister(unpack(arg))
2378
+ end
2379
+
2380
+ ReqlessAPI["queue.jobsByState"] = function(now, state, ...)
2381
+ local result = Reqless.jobs(now, state, unpack(arg))
2382
+ return cjsonArrayDegenerationWorkaround(result)
2383
+ end
2384
+
2385
+ ReqlessAPI['queue.length'] = function(now, queue)
2386
+ return Reqless.queue(queue):length()
2387
+ end
2388
+
2389
+ ReqlessAPI['queue.pause'] = function(now, ...)
2390
+ ReqlessQueue.pause(now, unpack(arg))
2391
+ end
2392
+
2393
+ ReqlessAPI['queue.peek'] = function(now, queue, offset, limit)
2394
+ local jids = Reqless.queue(queue):peek(now, offset, limit)
2395
+ local response = {}
2396
+ for _, jid in ipairs(jids) do
2397
+ table.insert(response, Reqless.job(jid):data())
2398
+ end
2399
+ return cjsonArrayDegenerationWorkaround(response)
2400
+ end
2401
+
2402
+ ReqlessAPI['queue.pop'] = function(now, queue, worker, limit)
2403
+ local jids = Reqless.queue(queue):pop(now, worker, limit)
2404
+ local response = {}
2405
+ for _, jid in ipairs(jids) do
2406
+ table.insert(response, Reqless.job(jid):data())
2407
+ end
2408
+ return cjsonArrayDegenerationWorkaround(response)
2409
+ end
2410
+
2411
+ ReqlessAPI['queue.put'] = function(now, worker, queue, jid, klass, data, delay, ...)
2412
+ return Reqless.queue(queue):put(now, worker, jid, klass, data, delay, unpack(arg))
2413
+ end
2414
+
2415
+ ReqlessAPI['queue.recurAtInterval'] = function(now, queue, jid, klass, data, interval, offset, ...)
2416
+ return Reqless.queue(queue):recurAtInterval(now, jid, klass, data, interval, offset, unpack(arg))
2417
+ end
2418
+
2419
+ ReqlessAPI['queue.stats'] = function(now, queue, date)
2420
+ return cjson.encode(Reqless.queue(queue):stats(now, date))
2421
+ end
2422
+
2423
+ ReqlessAPI['queue.throttle.get'] = function(now, queue)
2424
+ return ReqlessAPI['throttle.get'](now, ReqlessQueue.ns .. queue)
2425
+ end
2426
+
2427
+ ReqlessAPI['queue.throttle.set'] = function(now, queue, max)
2428
+ Reqless.throttle(ReqlessQueue.ns .. queue):set({maximum = max}, 0)
2429
+ end
2430
+
2431
+ ReqlessAPI['queue.unfail'] = function(now, queue, group, limit)
2432
+ assert(queue, 'queue.unfail(): Arg "queue" missing')
2433
+ return Reqless.queue(queue):unfail(now, group, limit)
2434
+ end
2435
+
2436
+ ReqlessAPI['queue.unpause'] = function(now, ...)
2437
+ ReqlessQueue.unpause(unpack(arg))
2438
+ end
2439
+
2440
+ ReqlessAPI['queueIdentifierPatterns.getAll'] = function(now)
2441
+ return cjson.encode(ReqlessQueuePatterns.getIdentifierPatterns(now))
2442
+ end
2443
+
2444
+ ReqlessAPI['queueIdentifierPatterns.setAll'] = function(now, ...)
2445
+ ReqlessQueuePatterns.setIdentifierPatterns(now, unpack(arg))
2446
+ end
2447
+
2448
+ ReqlessAPI['queuePriorityPatterns.getAll'] = function(now)
2449
+ return cjsonArrayDegenerationWorkaround(ReqlessQueuePatterns.getPriorityPatterns(now))
2450
+ end
2451
+
2452
+ ReqlessAPI['queuePriorityPatterns.setAll'] = function(now, ...)
2453
+ ReqlessQueuePatterns.setPriorityPatterns(now, unpack(arg))
2454
+ end
2455
+
2456
+ ReqlessAPI['queues.counts'] = function(now)
2457
+ return cjsonArrayDegenerationWorkaround(ReqlessQueue.counts(now, nil))
2458
+ end
2459
+
2460
+ ReqlessAPI['recurringJob.cancel'] = function(now, jid)
2461
+ return Reqless.recurring(jid):cancel()
2462
+ end
2463
+
2464
+ ReqlessAPI['recurringJob.get'] = function(now, jid)
2465
+ local data = Reqless.recurring(jid):data()
2466
+ if data then
2467
+ return cjson.encode(data)
2468
+ end
2469
+ end
2470
+
2471
+ ReqlessAPI['recurringJob.addTag'] = function(now, jid, ...)
2472
+ return Reqless.recurring(jid):tag(unpack(arg))
2473
+ end
2474
+
2475
+ ReqlessAPI['recurringJob.removeTag'] = function(now, jid, ...)
2476
+ return Reqless.recurring(jid):untag(unpack(arg))
2477
+ end
2478
+
2479
+ ReqlessAPI['recurringJob.update'] = function(now, jid, ...)
2480
+ return Reqless.recurring(jid):update(now, unpack(arg))
2481
+ end
2482
+
2483
+ ReqlessAPI['tags.top'] = function(now, offset, limit)
2484
+ local result = Reqless.tag(now, 'top', offset, limit)
2485
+ return cjsonArrayDegenerationWorkaround(result)
2486
+ end
2487
+
2488
+ ReqlessAPI['throttle.delete'] = function(now, tid)
2489
+ Reqless.throttle(tid):unset()
2490
+ end
2491
+
2492
+ ReqlessAPI['throttle.get'] = function(now, tid)
2493
+ return cjson.encode(Reqless.throttle(tid):dataWithTtl())
2494
+ end
2495
+
2496
+ ReqlessAPI['throttle.locks'] = function(now, tid)
2497
+ local result = Reqless.throttle(tid).locks.members()
2498
+ return cjsonArrayDegenerationWorkaround(result)
2499
+ end
2500
+
2501
+ ReqlessAPI['throttle.pending'] = function(now, tid)
2502
+ local result = Reqless.throttle(tid).pending.members()
2503
+ return cjsonArrayDegenerationWorkaround(result)
2504
+ end
2505
+
2506
+ ReqlessAPI['throttle.release'] = function(now, tid, ...)
2507
+ local throttle = Reqless.throttle(tid)
2508
+
2509
+ for _, jid in ipairs(arg) do
2510
+ throttle:release(now, jid)
2511
+ end
2512
+ end
2513
+
2514
+ ReqlessAPI['throttle.set'] = function(now, tid, max, ...)
2515
+ local expiration = unpack(arg)
2516
+ local data = {
2517
+ maximum = max
2518
+ }
2519
+ Reqless.throttle(tid):set(data, tonumber(expiration or 0))
2520
+ end
2521
+
2522
+ ReqlessAPI['worker.forget'] = function(now, ...)
2523
+ ReqlessWorker.deregister(unpack(arg))
2524
+ end
2525
+
2526
+ ReqlessAPI['worker.jobs'] = function(now, worker)
2527
+ return cjson.encode(ReqlessWorker.counts(now, worker))
2528
+ end
2529
+
2530
+ ReqlessAPI['workers.counts'] = function(now)
2531
+ return cjsonArrayDegenerationWorkaround(ReqlessWorker.counts(now, nil))
2532
+ end
2533
+
2534
+
2535
+ if #KEYS > 0 then error('No Keys should be provided') end
2536
+
2537
+ local command_name = assert(table.remove(ARGV, 1), 'Must provide a command')
2538
+ local command = assert(
2539
+ ReqlessAPI[command_name], 'Unknown command ' .. command_name)
2540
+
2541
+ local now = tonumber(table.remove(ARGV, 1))
2542
+ local now = assert(
2543
+ now, 'Arg "now" missing or not a number: ' .. (now or 'nil'))
2544
+
2545
+ return command(now, unpack(ARGV))