qless 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d90cb39f5a815e14fe5213ee01ec5ea36b771d02
4
- data.tar.gz: 1bbe259be1439258e67a380ddf5a9135395123e8
3
+ metadata.gz: 3904eda7b2060bae62e4859fa594826f950c8119
4
+ data.tar.gz: 471a898995adbdb1de64be7b5fbf0f8dc86629b4
5
5
  SHA512:
6
- metadata.gz: 997edf93ad4f9b4fd87bdf2058e4355bf995c0fceba227006b31658d69acdb274f85347baa0fa562e85a9a1c540187caeae0ec9e33494ea10cfcf2a3e0531e8b
7
- data.tar.gz: acac2dc97737aa5b63785aab8dec49dc688c6f55f1a6abd447e9d0450be3ac274dd261fff6505c7eef97b7a394c100dfdff88b873e11a37f74adb1c4c1b27aeb
6
+ metadata.gz: edf869cddce2b2e31c0bb01664c706409c49c673eaa99eba0f804fa387c9a6d74af546116ea863b5a1945d18d17f59e99f296fe9d83ca0be389f6e8210902d20
7
+ data.tar.gz: 3efaa022a3687c87f6e2db7cf5abc60afa2eb94141861f21d7d57d1f8d26ca9a8806341c52de9f347700d7503bae84e6ee1433ace899e08eeddbea35baa11821
data/README.md CHANGED
@@ -663,3 +663,12 @@ Mailing List
663
663
 
664
664
  For questions and general Qless discussion, please join the [Qless
665
665
  Mailing list](https://groups.google.com/forum/?fromgroups#!forum/qless).
666
+
667
+ Release Notes
668
+ =============
669
+
670
+ 0.12.0
671
+ ------
672
+ The metric `failures` provided by `qless-stats` has been replaced by `failed` for
673
+ compatibility with users of `graphite`. See [#275](https://github.com/seomoz/qless/pull/275)
674
+ for more details.
@@ -96,7 +96,7 @@ class Stats < Thor
96
96
  client.gauge "failures.#{failure}", count
97
97
  count
98
98
  end.reduce(0, :+)
99
- client.gauge 'failures', total
99
+ client.gauge 'failed', total
100
100
 
101
101
  # Track workers
102
102
  client.gauge 'workers', qless.workers.counts.length
@@ -1,4 +1,4 @@
1
- -- Current SHA: 525c39000dc71df53a3502491cb4daf0e1128f1d
1
+ -- Current SHA: 9d2cca3846a96fee53000085e36638e74ed392ed
2
2
  -- This is a generated file
3
3
  -------------------------------------------------------------------------------
4
4
  -- Forward declarations to make everything happy
@@ -68,14 +68,14 @@ end
68
68
  -- If no group is provided, this returns a JSON blob of the counts of the
69
69
  -- various groups of failures known. If a group is provided, it will report up
70
70
  -- to `limit` from `start` of the jobs affected by that issue.
71
- --
71
+ --
72
72
  -- # If no group, then...
73
73
  -- {
74
74
  -- 'group1': 1,
75
75
  -- 'group2': 5,
76
76
  -- ...
77
77
  -- }
78
- --
78
+ --
79
79
  -- # If a group is provided, then...
80
80
  -- {
81
81
  -- 'total': 20,
@@ -121,9 +121,9 @@ end
121
121
  -------------------------------------------------------------------------------
122
122
  -- Return all the job ids currently considered to be in the provided state
123
123
  -- in a particular queue. The response is a list of job ids:
124
- --
124
+ --
125
125
  -- [
126
- -- jid1,
126
+ -- jid1,
127
127
  -- jid2,
128
128
  -- ...
129
129
  -- ]
@@ -154,7 +154,7 @@ function Qless.jobs(now, state, ...)
154
154
  elseif state == 'depends' then
155
155
  return queue.depends.peek(now, offset, count)
156
156
  elseif state == 'recurring' then
157
- return queue.recurring.peek(math.huge, offset, count)
157
+ return queue.recurring.peek('+inf', offset, count)
158
158
  else
159
159
  error('Jobs(): Unknown type "' .. state .. '"')
160
160
  end
@@ -169,7 +169,7 @@ end
169
169
  -- associated with that id, and 'untrack' stops tracking it. In this context,
170
170
  -- tracking is nothing more than saving the job to a list of jobs that are
171
171
  -- considered special.
172
- --
172
+ --
173
173
  -- {
174
174
  -- 'jobs': [
175
175
  -- {
@@ -254,18 +254,18 @@ function Qless.tag(now, command, ...)
254
254
  tags = cjson.decode(tags)
255
255
  local _tags = {}
256
256
  for i,v in ipairs(tags) do _tags[v] = true end
257
-
257
+
258
258
  -- Otherwise, add the job to the sorted set with that tags
259
259
  for i=2,#arg do
260
260
  local tag = arg[i]
261
- if _tags[tag] == nil then
261
+ if _tags[tag] == nil or _tags[tag] == false then
262
262
  _tags[tag] = true
263
263
  table.insert(tags, tag)
264
264
  end
265
265
  redis.call('zadd', 'ql:t:' .. tag, now, jid)
266
266
  redis.call('zincrby', 'ql:tags', 1, tag)
267
267
  end
268
-
268
+
269
269
  redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(tags))
270
270
  return tags
271
271
  else
@@ -280,7 +280,7 @@ function Qless.tag(now, command, ...)
280
280
  tags = cjson.decode(tags)
281
281
  local _tags = {}
282
282
  for i,v in ipairs(tags) do _tags[v] = true end
283
-
283
+
284
284
  -- Otherwise, add the job to the sorted set with that tags
285
285
  for i=2,#arg do
286
286
  local tag = arg[i]
@@ -288,10 +288,10 @@ function Qless.tag(now, command, ...)
288
288
  redis.call('zrem', 'ql:t:' .. tag, jid)
289
289
  redis.call('zincrby', 'ql:tags', -1, tag)
290
290
  end
291
-
291
+
292
292
  local results = {}
293
293
  for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
294
-
294
+
295
295
  redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(results))
296
296
  return results
297
297
  else
@@ -333,7 +333,7 @@ function Qless.cancel(...)
333
333
  -- make sure that this operation will be ok
334
334
  for i, jid in ipairs(arg) do
335
335
  for j, dep in ipairs(dependents[jid]) do
336
- if dependents[dep] == nil then
336
+ if dependents[dep] == nil or dependents[dep] == false then
337
337
  error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep ..
338
338
  ' but is not mentioned to be canceled')
339
339
  end
@@ -418,7 +418,7 @@ function Qless.cancel(...)
418
418
  redis.call('del', QlessJob.ns .. jid .. '-history')
419
419
  end
420
420
  end
421
-
421
+
422
422
  return arg
423
423
  end
424
424
 
@@ -535,26 +535,26 @@ end
535
535
 
536
536
  -- Complete a job and optionally put it in another queue, either scheduled or
537
537
  -- to be considered waiting immediately. It can also optionally accept other
538
- -- jids on which this job will be considered dependent before it's considered
538
+ -- jids on which this job will be considered dependent before it's considered
539
539
  -- valid.
540
540
  --
541
541
  -- The variable-length arguments may be pairs of the form:
542
- --
542
+ --
543
543
  -- ('next' , queue) : The queue to advance it to next
544
544
  -- ('delay' , delay) : The delay for the next queue
545
545
  -- ('depends', : Json of jobs it depends on in the new queue
546
546
  -- '["jid1", "jid2", ...]')
547
547
  ---
548
- function QlessJob:complete(now, worker, queue, data, ...)
548
+ function QlessJob:complete(now, worker, queue, raw_data, ...)
549
549
  assert(worker, 'Complete(): Arg "worker" missing')
550
550
  assert(queue , 'Complete(): Arg "queue" missing')
551
- data = assert(cjson.decode(data),
552
- 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data))
551
+ local data = assert(cjson.decode(raw_data),
552
+ 'Complete(): Arg "data" missing or not JSON: ' .. tostring(raw_data))
553
553
 
554
554
  -- Read in all the optional parameters
555
555
  local options = {}
556
556
  for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
557
-
557
+
558
558
  -- Sanity check on optional args
559
559
  local nextq = options['next']
560
560
  local delay = assert(tonumber(options['delay'] or 0))
@@ -581,14 +581,15 @@ function QlessJob:complete(now, worker, queue, data, ...)
581
581
  'priority', 'retries', 'queue'))
582
582
 
583
583
  if lastworker == false then
584
- error('Complete(): Job does not exist')
584
+ error('Complete(): Job ' .. self.jid .. ' does not exist')
585
585
  elseif (state ~= 'running') then
586
- error('Complete(): Job is not currently running: ' .. state)
586
+ error('Complete(): Job ' .. self.jid .. ' is not currently running: ' ..
587
+ state)
587
588
  elseif lastworker ~= worker then
588
- error('Complete(): Job has been handed out to another worker: ' ..
589
- tostring(lastworker))
589
+ error('Complete(): Job ' .. self.jid ..
590
+ ' has been handed out to another worker: ' .. tostring(lastworker))
590
591
  elseif queue ~= current_queue then
591
- error('Complete(): Job running in another queue: ' ..
592
+ error('Complete(): Job ' .. self.jid .. ' running in another queue: ' ..
592
593
  tostring(current_queue))
593
594
  end
594
595
 
@@ -600,8 +601,8 @@ function QlessJob:complete(now, worker, queue, data, ...)
600
601
  -- update history
601
602
  self:history(now, 'done')
602
603
 
603
- if data then
604
- redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
604
+ if raw_data then
605
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', raw_data)
605
606
  end
606
607
 
607
608
  -- Remove the job from the previous queue
@@ -647,7 +648,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
647
648
  if redis.call('zscore', 'ql:queues', nextq) == false then
648
649
  redis.call('zadd', 'ql:queues', now, nextq)
649
650
  end
650
-
651
+
651
652
  redis.call('hmset', QlessJob.ns .. self.jid,
652
653
  'state', 'waiting',
653
654
  'worker', '',
@@ -655,7 +656,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
655
656
  'queue', nextq,
656
657
  'expires', 0,
657
658
  'remaining', tonumber(retries))
658
-
659
+
659
660
  if (delay > 0) and (#depends == 0) then
660
661
  queue_obj.scheduled.add(now + delay, self.jid)
661
662
  return 'scheduled'
@@ -703,18 +704,18 @@ function QlessJob:complete(now, worker, queue, data, ...)
703
704
  'queue', '',
704
705
  'expires', 0,
705
706
  'remaining', tonumber(retries))
706
-
707
+
707
708
  -- Do the completion dance
708
709
  local count = Qless.config.get('jobs-history-count')
709
710
  local time = Qless.config.get('jobs-history')
710
-
711
+
711
712
  -- These are the default values
712
713
  count = tonumber(count or 50000)
713
714
  time = tonumber(time or 7 * 24 * 60 * 60)
714
-
715
+
715
716
  -- Schedule this job for destructination eventually
716
717
  redis.call('zadd', 'ql:completed', now, self.jid)
717
-
718
+
718
719
  -- Now look at the expired job data. First, based on the current time
719
720
  local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
720
721
  -- Any jobs that need to be expired... delete
@@ -730,7 +731,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
730
731
  end
731
732
  -- And now remove those from the queued-for-cleanup queue
732
733
  redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
733
-
734
+
734
735
  -- Now take the all by the most recent 'count' ids
735
736
  jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
736
737
  for index, jid in ipairs(jids) do
@@ -744,7 +745,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
744
745
  redis.call('del', QlessJob.ns .. jid .. '-history')
745
746
  end
746
747
  redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
747
-
748
+
748
749
  -- Alright, if this has any dependents, then we should go ahead
749
750
  -- and unstick those guys.
750
751
  for i, j in ipairs(redis.call(
@@ -768,10 +769,10 @@ function QlessJob:complete(now, worker, queue, data, ...)
768
769
  end
769
770
  end
770
771
  end
771
-
772
+
772
773
  -- Delete our dependents key
773
774
  redis.call('del', QlessJob.ns .. self.jid .. '-dependents')
774
-
775
+
775
776
  return 'complete'
776
777
  end
777
778
  end
@@ -782,14 +783,14 @@ end
782
783
  -- specific message. By `group`, we mean some phrase that might be one of
783
784
  -- several categorical modes of failure. The `message` is something more
784
785
  -- job-specific, like perhaps a traceback.
785
- --
786
+ --
786
787
  -- This method should __not__ be used to note that a job has been dropped or
787
788
  -- has failed in a transient way. This method __should__ be used to note that
788
789
  -- a job has something really wrong with it that must be remedied.
789
- --
790
+ --
790
791
  -- The motivation behind the `group` is so that similar errors can be grouped
791
792
  -- together. Optionally, updated data can be provided for the job. A job in
792
- -- any state can be marked as failed. If it has been given to a worker as a
793
+ -- any state can be marked as failed. If it has been given to a worker as a
793
794
  -- job, then its subsequent requests to heartbeat or complete that job will
794
795
  -- fail. Failed jobs are kept until they are canceled or completed.
795
796
  --
@@ -821,11 +822,12 @@ function QlessJob:fail(now, worker, group, message, data)
821
822
 
822
823
  -- If the job has been completed, we cannot fail it
823
824
  if not state then
824
- error('Fail(): Job does not exist')
825
+ error('Fail(): Job ' .. self.jid .. 'does not exist')
825
826
  elseif state ~= 'running' then
826
- error('Fail(): Job not currently running: ' .. state)
827
+ error('Fail(): Job ' .. self.jid .. 'not currently running: ' .. state)
827
828
  elseif worker ~= oldworker then
828
- error('Fail(): Job running with another worker: ' .. oldworker)
829
+ error('Fail(): Job ' .. self.jid .. ' running with another worker: ' ..
830
+ oldworker)
829
831
  end
830
832
 
831
833
  -- Send out a log message
@@ -860,7 +862,7 @@ function QlessJob:fail(now, worker, group, message, data)
860
862
  queue_obj.locks.remove(self.jid)
861
863
  queue_obj.scheduled.remove(self.jid)
862
864
 
863
- -- The reason that this appears here is that the above will fail if the
865
+ -- The reason that this appears here is that the above will fail if the
864
866
  -- job doesn't exist
865
867
  if data then
866
868
  redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
@@ -897,7 +899,7 @@ end
897
899
  -- Throws an exception if:
898
900
  -- - the worker is not the worker with a lock on the job
899
901
  -- - the job is not actually running
900
- --
902
+ --
901
903
  -- Otherwise, it returns the number of retries remaining. If the allowed
902
904
  -- retries have been exhausted, then it is automatically failed, and a negative
903
905
  -- number is returned.
@@ -910,7 +912,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
910
912
  assert(worker, 'Retry(): Arg "worker" missing')
911
913
  delay = assert(tonumber(delay or 0),
912
914
  'Retry(): Arg "delay" not a number: ' .. tostring(delay))
913
-
915
+
914
916
  -- Let's see what the old priority, and tags were
915
917
  local oldqueue, state, retries, oldworker, priority, failure = unpack(
916
918
  redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state',
@@ -918,11 +920,13 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
918
920
 
919
921
  -- If this isn't the worker that owns
920
922
  if oldworker == false then
921
- error('Retry(): Job does not exist')
923
+ error('Retry(): Job ' .. self.jid .. ' does not exist')
922
924
  elseif state ~= 'running' then
923
- error('Retry(): Job is not currently running: ' .. state)
925
+ error('Retry(): Job ' .. self.jid .. ' is not currently running: ' ..
926
+ state)
924
927
  elseif oldworker ~= worker then
925
- error('Retry(): Job has been given to another worker: ' .. oldworker)
928
+ error('Retry(): Job ' .. self.jid ..
929
+ ' has been given to another worker: ' .. oldworker)
926
930
  end
927
931
 
928
932
  -- For each of these, decrement their retries. If any of them
@@ -943,7 +947,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
943
947
  -- queue it's in
944
948
  local group = group or 'failed-retries-' .. queue
945
949
  self:history(now, 'failed', {['group'] = group})
946
-
950
+
947
951
  redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed',
948
952
  'worker', '',
949
953
  'expires', '')
@@ -967,7 +971,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
967
971
  ['worker'] = unpack(self:data('worker'))
968
972
  }))
969
973
  end
970
-
974
+
971
975
  -- Add this type of failure to the list of failures
972
976
  redis.call('sadd', 'ql:failures', group)
973
977
  -- And add this particular instance to the failed types
@@ -1103,11 +1107,14 @@ function QlessJob:heartbeat(now, worker, data)
1103
1107
  redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state'))
1104
1108
  if job_worker == false then
1105
1109
  -- This means the job doesn't exist
1106
- error('Heartbeat(): Job does not exist')
1110
+ error('Heartbeat(): Job ' .. self.jid .. ' does not exist')
1107
1111
  elseif state ~= 'running' then
1108
- error('Heartbeat(): Job not currently running: ' .. state)
1112
+ error(
1113
+ 'Heartbeat(): Job ' .. self.jid .. ' not currently running: ' .. state)
1109
1114
  elseif job_worker ~= worker or #job_worker == 0 then
1110
- error('Heartbeat(): Job given out to another worker: ' .. job_worker)
1115
+ error(
1116
+ 'Heartbeat(): Job ' .. self.jid ..
1117
+ ' given out to another worker: ' .. job_worker)
1111
1118
  else
1112
1119
  -- Otherwise, optionally update the user data, and the heartbeat
1113
1120
  if data then
@@ -1119,11 +1126,11 @@ function QlessJob:heartbeat(now, worker, data)
1119
1126
  redis.call('hmset', QlessJob.ns .. self.jid,
1120
1127
  'expires', expires, 'worker', worker)
1121
1128
  end
1122
-
1129
+
1123
1130
  -- Update hwen this job was last updated on that worker
1124
1131
  -- Add this job to the list of jobs handled by this worker
1125
1132
  redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
1126
-
1133
+
1127
1134
  -- And now we should just update the locks
1128
1135
  local queue = Qless.queue(
1129
1136
  redis.call('hget', QlessJob.ns .. self.jid, 'queue'))
@@ -1144,7 +1151,7 @@ function QlessJob:priority(priority)
1144
1151
  -- Get the queue the job is currently in, if any
1145
1152
  local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue')
1146
1153
 
1147
- if queue == nil then
1154
+ if queue == nil or queue == false then
1148
1155
  -- If the job doesn't exist, throw an error
1149
1156
  error('Priority(): Job ' .. self.jid .. ' does not exist')
1150
1157
  elseif queue == '' then
@@ -1177,8 +1184,8 @@ end
1177
1184
  function QlessJob:timeout(now)
1178
1185
  local queue_name, state, worker = unpack(redis.call('hmget',
1179
1186
  QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
1180
- if queue_name == nil then
1181
- error('Timeout(): Job does not exist')
1187
+ if queue_name == nil or queue_name == false then
1188
+ error('Timeout(): Job ' .. self.jid .. ' does not exist')
1182
1189
  elseif state ~= 'running' then
1183
1190
  error('Timeout(): Job ' .. self.jid .. ' not running')
1184
1191
  else
@@ -1186,7 +1193,7 @@ function QlessJob:timeout(now)
1186
1193
  self:history(now, 'timed-out')
1187
1194
  local queue = Qless.queue(queue_name)
1188
1195
  queue.locks.remove(self.jid)
1189
- queue.work.add(now, math.huge, self.jid)
1196
+ queue.work.add(now, '+inf', self.jid)
1190
1197
  redis.call('hmset', QlessJob.ns .. self.jid,
1191
1198
  'state', 'stalled', 'expires', 0)
1192
1199
  local encoded = cjson.encode({
@@ -1261,7 +1268,7 @@ function QlessJob:history(now, what, item)
1261
1268
  -- We'll always keep the first item around
1262
1269
  local obj = redis.call('lpop', QlessJob.ns .. self.jid .. '-history')
1263
1270
  redis.call('ltrim', QlessJob.ns .. self.jid .. '-history', -count + 2, -1)
1264
- if obj ~= nil then
1271
+ if obj ~= nil and obj ~= false then
1265
1272
  redis.call('lpush', QlessJob.ns .. self.jid .. '-history', obj)
1266
1273
  end
1267
1274
  end
@@ -1296,8 +1303,11 @@ function Qless.queue(name)
1296
1303
  return redis.call('zrem', queue:prefix('work'), unpack(arg))
1297
1304
  end
1298
1305
  end, add = function(now, priority, jid)
1306
+ if priority ~= '+inf' then
1307
+ priority = priority - (now / 10000000000)
1308
+ end
1299
1309
  return redis.call('zadd',
1300
- queue:prefix('work'), priority - (now / 10000000000), jid)
1310
+ queue:prefix('work'), priority, jid)
1301
1311
  end, score = function(jid)
1302
1312
  return redis.call('zscore', queue:prefix('work'), jid)
1303
1313
  end, length = function()
@@ -1309,10 +1319,10 @@ function Qless.queue(name)
1309
1319
  queue.locks = {
1310
1320
  expired = function(now, offset, count)
1311
1321
  return redis.call('zrangebyscore',
1312
- queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count)
1322
+ queue:prefix('locks'), '-inf', now, 'LIMIT', offset, count)
1313
1323
  end, peek = function(now, offset, count)
1314
1324
  return redis.call('zrangebyscore', queue:prefix('locks'),
1315
- now, math.huge, 'LIMIT', offset, count)
1325
+ now, '+inf', 'LIMIT', offset, count)
1316
1326
  end, add = function(expires, jid)
1317
1327
  redis.call('zadd', queue:prefix('locks'), expires, jid)
1318
1328
  end, remove = function(...)
@@ -1320,7 +1330,7 @@ function Qless.queue(name)
1320
1330
  return redis.call('zrem', queue:prefix('locks'), unpack(arg))
1321
1331
  end
1322
1332
  end, running = function(now)
1323
- return redis.call('zcount', queue:prefix('locks'), now, math.huge)
1333
+ return redis.call('zcount', queue:prefix('locks'), now, '+inf')
1324
1334
  end, length = function(now)
1325
1335
  -- If a 'now' is provided, we're interested in how many are before
1326
1336
  -- that time
@@ -1453,11 +1463,11 @@ function QlessQueue:stats(now, date)
1453
1463
 
1454
1464
  local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
1455
1465
  local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
1456
-
1466
+
1457
1467
  count = tonumber(count) or 0
1458
1468
  mean = tonumber(mean) or 0
1459
1469
  vk = tonumber(vk)
1460
-
1470
+
1461
1471
  results.count = count or 0
1462
1472
  results.mean = mean or 0
1463
1473
  results.histogram = {}
@@ -1507,8 +1517,8 @@ function QlessQueue:peek(now, count)
1507
1517
 
1508
1518
  -- Now we've checked __all__ the locks for this queue the could
1509
1519
  -- have expired, and are no more than the number requested. If
1510
- -- we still need values in order to meet the demand, then we
1511
- -- should check if any scheduled items, and if so, we should
1520
+ -- we still need values in order to meet the demand, then we
1521
+ -- should check if any scheduled items, and if so, we should
1512
1522
  -- insert them to ensure correctness when pulling off the next
1513
1523
  -- unit of work.
1514
1524
  self:check_scheduled(now, count - #jids)
@@ -1582,8 +1592,8 @@ function QlessQueue:pop(now, worker, count)
1582
1592
  -- look for all the recurring jobs that need jobs run
1583
1593
  self:check_recurring(now, count - #jids)
1584
1594
 
1585
- -- If we still need values in order to meet the demand, then we
1586
- -- should check if any scheduled items, and if so, we should
1595
+ -- If we still need values in order to meet the demand, then we
1596
+ -- should check if any scheduled items, and if so, we should
1587
1597
  -- insert them to ensure correctness when pulling off the next
1588
1598
  -- unit of work.
1589
1599
  self:check_scheduled(now, count - #jids)
@@ -1605,19 +1615,19 @@ function QlessQueue:pop(now, worker, count)
1605
1615
  self:stat(now, 'wait', waiting)
1606
1616
  redis.call('hset', QlessJob.ns .. jid,
1607
1617
  'time', string.format("%.20f", now))
1608
-
1618
+
1609
1619
  -- Add this job to the list of jobs handled by this worker
1610
1620
  redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
1611
-
1621
+
1612
1622
  -- Update the jobs data, and add its locks, and return the job
1613
1623
  job:update({
1614
1624
  worker = worker,
1615
1625
  expires = expires,
1616
1626
  state = 'running'
1617
1627
  })
1618
-
1628
+
1619
1629
  self.locks.add(expires, jid)
1620
-
1630
+
1621
1631
  local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
1622
1632
  if tracked then
1623
1633
  Qless.publish('popped', jid)
@@ -1668,7 +1678,7 @@ function QlessQueue:stat(now, stat, val)
1668
1678
  redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
1669
1679
  else -- days
1670
1680
  redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
1671
- end
1681
+ end
1672
1682
  redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
1673
1683
  end
1674
1684
 
@@ -1728,8 +1738,8 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
1728
1738
  -- Now find what's in the original, but not the new
1729
1739
  local original = redis.call(
1730
1740
  'smembers', QlessJob.ns .. jid .. '-dependencies')
1731
- for _, dep in pairs(original) do
1732
- if new[dep] == nil then
1741
+ for _, dep in pairs(original) do
1742
+ if new[dep] == nil or new[dep] == false then
1733
1743
  -- Remove k as a dependency
1734
1744
  redis.call('srem', QlessJob.ns .. dep .. '-dependents' , jid)
1735
1745
  redis.call('srem', QlessJob.ns .. jid .. '-dependencies', dep)
@@ -1851,7 +1861,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
1851
1861
  end
1852
1862
 
1853
1863
  -- Lastly, we're going to make sure that this item is in the
1854
- -- set of known queues. We should keep this sorted by the
1864
+ -- set of known queues. We should keep this sorted by the
1855
1865
  -- order in which we saw each of these queues
1856
1866
  if redis.call('zscore', 'ql:queues', self.name) == false then
1857
1867
  redis.call('zadd', 'ql:queues', now, self.name)
@@ -1921,7 +1931,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1921
1931
  if #arg % 2 == 1 then
1922
1932
  error('Odd number of additional args: ' .. tostring(arg))
1923
1933
  end
1924
-
1934
+
1925
1935
  -- Read in all the optional parameters
1926
1936
  local options = {}
1927
1937
  for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end
@@ -1941,12 +1951,12 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1941
1951
  local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
1942
1952
  count = count or 0
1943
1953
 
1944
- -- If it has previously been in another queue, then we should remove
1954
+ -- If it has previously been in another queue, then we should remove
1945
1955
  -- some information about it
1946
1956
  if old_queue then
1947
1957
  Qless.queue(old_queue).recurring.remove(jid)
1948
1958
  end
1949
-
1959
+
1950
1960
  -- Do some insertions
1951
1961
  redis.call('hmset', 'ql:r:' .. jid,
1952
1962
  'jid' , jid,
@@ -1964,14 +1974,14 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1964
1974
  'backlog' , options.backlog)
1965
1975
  -- Now, we should schedule the next run of the job
1966
1976
  self.recurring.add(now + offset, jid)
1967
-
1977
+
1968
1978
  -- Lastly, we're going to make sure that this item is in the
1969
- -- set of known queues. We should keep this sorted by the
1979
+ -- set of known queues. We should keep this sorted by the
1970
1980
  -- order in which we saw each of these queues
1971
1981
  if redis.call('zscore', 'ql:queues', self.name) == false then
1972
1982
  redis.call('zadd', 'ql:queues', now, self.name)
1973
1983
  end
1974
-
1984
+
1975
1985
  return jid
1976
1986
  else
1977
1987
  error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
@@ -2017,22 +2027,22 @@ function QlessQueue:check_recurring(now, count)
2017
2027
  )
2018
2028
  end
2019
2029
  end
2020
-
2021
- -- We're saving this value so that in the history, we can accurately
2030
+
2031
+ -- We're saving this value so that in the history, we can accurately
2022
2032
  -- reflect when the job would normally have been scheduled
2023
2033
  while (score <= now) and (moved < count) do
2024
2034
  local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
2025
2035
  moved = moved + 1
2026
2036
 
2027
2037
  local child_jid = jid .. '-' .. count
2028
-
2038
+
2029
2039
  -- Add this job to the list of jobs tagged with whatever tags were
2030
2040
  -- supplied
2031
2041
  for i, tag in ipairs(_tags) do
2032
2042
  redis.call('zadd', 'ql:t:' .. tag, now, child_jid)
2033
2043
  redis.call('zincrby', 'ql:tags', 1, tag)
2034
2044
  end
2035
-
2045
+
2036
2046
  -- First, let's save its data
2037
2047
  redis.call('hmset', QlessJob.ns .. child_jid,
2038
2048
  'jid' , child_jid,
@@ -2049,12 +2059,12 @@ function QlessQueue:check_recurring(now, count)
2049
2059
  'time' , string.format("%.20f", score),
2050
2060
  'spawned_from_jid', jid)
2051
2061
  Qless.job(child_jid):history(score, 'put', {q = self.name})
2052
-
2062
+
2053
2063
  -- Now, if a delay was provided, and if it's in the future,
2054
2064
  -- then we'll have to schedule it. Otherwise, we're just
2055
2065
  -- going to add it to the work queue.
2056
2066
  self.work.add(score, priority, child_jid)
2057
-
2067
+
2058
2068
  score = score + interval
2059
2069
  self.recurring.add(score, jid)
2060
2070
  end
@@ -2069,7 +2079,7 @@ function QlessQueue:check_scheduled(now, count)
2069
2079
  -- insert into the work queue
2070
2080
  local scheduled = self.scheduled.ready(now, 0, count)
2071
2081
  for index, jid in ipairs(scheduled) do
2072
- -- With these in hand, we'll have to go out and find the
2082
+ -- With these in hand, we'll have to go out and find the
2073
2083
  -- priorities of these jobs, and then we'll insert them
2074
2084
  -- into the work queue and then when that's complete, we'll
2075
2085
  -- remove them from the scheduled queue
@@ -2154,7 +2164,7 @@ function QlessQueue:invalidate_locks(now, count)
2154
2164
  -- See how many remaining retries the job has
2155
2165
  local remaining = tonumber(redis.call(
2156
2166
  'hincrby', QlessJob.ns .. jid, 'remaining', -1))
2157
-
2167
+
2158
2168
  -- This is where we actually have to time out the work
2159
2169
  if remaining < 0 then
2160
2170
  -- Now remove the instance from the schedule, and work queues
@@ -2162,7 +2172,7 @@ function QlessQueue:invalidate_locks(now, count)
2162
2172
  self.work.remove(jid)
2163
2173
  self.locks.remove(jid)
2164
2174
  self.scheduled.remove(jid)
2165
-
2175
+
2166
2176
  local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
2167
2177
  local job = Qless.job(jid)
2168
2178
  job:history(now, 'failed', {group = group})
@@ -2178,12 +2188,12 @@ function QlessQueue:invalidate_locks(now, count)
2178
2188
  ['when'] = now,
2179
2189
  ['worker'] = unpack(job:data('worker'))
2180
2190
  }))
2181
-
2191
+
2182
2192
  -- Add this type of failure to the list of failures
2183
2193
  redis.call('sadd', 'ql:failures', group)
2184
2194
  -- And add this particular instance to the failed types
2185
2195
  redis.call('lpush', 'ql:f:' .. group, jid)
2186
-
2196
+
2187
2197
  if redis.call('zscore', 'ql:tracked', jid) ~= false then
2188
2198
  Qless.publish('failed', jid)
2189
2199
  end
@@ -2260,11 +2270,11 @@ function QlessRecurringJob:data()
2260
2270
  local job = redis.call(
2261
2271
  'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue',
2262
2272
  'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog')
2263
-
2273
+
2264
2274
  if not job[1] then
2265
2275
  return nil
2266
2276
  end
2267
-
2277
+
2268
2278
  return {
2269
2279
  jid = job[1],
2270
2280
  klass = job[2],
@@ -2287,7 +2297,7 @@ end
2287
2297
  -- - data
2288
2298
  -- - klass
2289
2299
  -- - queue
2290
- -- - backlog
2300
+ -- - backlog
2291
2301
  function QlessRecurringJob:update(now, ...)
2292
2302
  local options = {}
2293
2303
  -- Make sure that the job exists
@@ -2345,10 +2355,10 @@ function QlessRecurringJob:tag(...)
2345
2355
  tags = cjson.decode(tags)
2346
2356
  local _tags = {}
2347
2357
  for i,v in ipairs(tags) do _tags[v] = true end
2348
-
2358
+
2349
2359
  -- Otherwise, add the job to the sorted set with that tags
2350
- for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end
2351
-
2360
+ for i=1,#arg do if _tags[arg[i]] == nil or _tags[arg[i]] == false then table.insert(tags, arg[i]) end end
2361
+
2352
2362
  tags = cjson.encode(tags)
2353
2363
  redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
2354
2364
  return tags
@@ -2404,7 +2414,7 @@ end
2404
2414
  -- Provide data about all the workers, or if a specific worker is provided,
2405
2415
  -- then which jobs that worker is responsible for. If no worker is provided,
2406
2416
  -- expect a response of the form:
2407
- --
2417
+ --
2408
2418
  -- [
2409
2419
  -- # This is sorted by the recency of activity from that worker
2410
2420
  -- {
@@ -2415,9 +2425,9 @@ end
2415
2425
  -- ...
2416
2426
  -- }
2417
2427
  -- ]
2418
- --
2428
+ --
2419
2429
  -- If a worker id is provided, then expect a response of the form:
2420
- --
2430
+ --
2421
2431
  -- {
2422
2432
  -- 'jobs': [
2423
2433
  -- jid1,
@@ -1,4 +1,4 @@
1
- -- Current SHA: 525c39000dc71df53a3502491cb4daf0e1128f1d
1
+ -- Current SHA: 9d2cca3846a96fee53000085e36638e74ed392ed
2
2
  -- This is a generated file
3
3
  local Qless = {
4
4
  ns = 'ql:'
@@ -98,7 +98,7 @@ function Qless.jobs(now, state, ...)
98
98
  elseif state == 'depends' then
99
99
  return queue.depends.peek(now, offset, count)
100
100
  elseif state == 'recurring' then
101
- return queue.recurring.peek(math.huge, offset, count)
101
+ return queue.recurring.peek('+inf', offset, count)
102
102
  else
103
103
  error('Jobs(): Unknown type "' .. state .. '"')
104
104
  end
@@ -147,17 +147,17 @@ function Qless.tag(now, command, ...)
147
147
  tags = cjson.decode(tags)
148
148
  local _tags = {}
149
149
  for i,v in ipairs(tags) do _tags[v] = true end
150
-
150
+
151
151
  for i=2,#arg do
152
152
  local tag = arg[i]
153
- if _tags[tag] == nil then
153
+ if _tags[tag] == nil or _tags[tag] == false then
154
154
  _tags[tag] = true
155
155
  table.insert(tags, tag)
156
156
  end
157
157
  redis.call('zadd', 'ql:t:' .. tag, now, jid)
158
158
  redis.call('zincrby', 'ql:tags', 1, tag)
159
159
  end
160
-
160
+
161
161
  redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(tags))
162
162
  return tags
163
163
  else
@@ -170,17 +170,17 @@ function Qless.tag(now, command, ...)
170
170
  tags = cjson.decode(tags)
171
171
  local _tags = {}
172
172
  for i,v in ipairs(tags) do _tags[v] = true end
173
-
173
+
174
174
  for i=2,#arg do
175
175
  local tag = arg[i]
176
176
  _tags[tag] = nil
177
177
  redis.call('zrem', 'ql:t:' .. tag, jid)
178
178
  redis.call('zincrby', 'ql:tags', -1, tag)
179
179
  end
180
-
180
+
181
181
  local results = {}
182
182
  for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
183
-
183
+
184
184
  redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(results))
185
185
  return results
186
186
  else
@@ -214,7 +214,7 @@ function Qless.cancel(...)
214
214
 
215
215
  for i, jid in ipairs(arg) do
216
216
  for j, dep in ipairs(dependents[jid]) do
217
- if dependents[dep] == nil then
217
+ if dependents[dep] == nil or dependents[dep] == false then
218
218
  error('Cancel(): ' .. jid .. ' is a dependency of ' .. dep ..
219
219
  ' but is not mentioned to be canceled')
220
220
  end
@@ -282,7 +282,7 @@ function Qless.cancel(...)
282
282
  redis.call('del', QlessJob.ns .. jid .. '-history')
283
283
  end
284
284
  end
285
-
285
+
286
286
  return arg
287
287
  end
288
288
 
@@ -376,15 +376,15 @@ function QlessJob:data(...)
376
376
  end
377
377
  end
378
378
 
379
- function QlessJob:complete(now, worker, queue, data, ...)
379
+ function QlessJob:complete(now, worker, queue, raw_data, ...)
380
380
  assert(worker, 'Complete(): Arg "worker" missing')
381
381
  assert(queue , 'Complete(): Arg "queue" missing')
382
- data = assert(cjson.decode(data),
383
- 'Complete(): Arg "data" missing or not JSON: ' .. tostring(data))
382
+ local data = assert(cjson.decode(raw_data),
383
+ 'Complete(): Arg "data" missing or not JSON: ' .. tostring(raw_data))
384
384
 
385
385
  local options = {}
386
386
  for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
387
-
387
+
388
388
  local nextq = options['next']
389
389
  local delay = assert(tonumber(options['delay'] or 0))
390
390
  local depends = assert(cjson.decode(options['depends'] or '[]'),
@@ -405,21 +405,22 @@ function QlessJob:complete(now, worker, queue, data, ...)
405
405
  'priority', 'retries', 'queue'))
406
406
 
407
407
  if lastworker == false then
408
- error('Complete(): Job does not exist')
408
+ error('Complete(): Job ' .. self.jid .. ' does not exist')
409
409
  elseif (state ~= 'running') then
410
- error('Complete(): Job is not currently running: ' .. state)
410
+ error('Complete(): Job ' .. self.jid .. ' is not currently running: ' ..
411
+ state)
411
412
  elseif lastworker ~= worker then
412
- error('Complete(): Job has been handed out to another worker: ' ..
413
- tostring(lastworker))
413
+ error('Complete(): Job ' .. self.jid ..
414
+ ' has been handed out to another worker: ' .. tostring(lastworker))
414
415
  elseif queue ~= current_queue then
415
- error('Complete(): Job running in another queue: ' ..
416
+ error('Complete(): Job ' .. self.jid .. ' running in another queue: ' ..
416
417
  tostring(current_queue))
417
418
  end
418
419
 
419
420
  self:history(now, 'done')
420
421
 
421
- if data then
422
- redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
422
+ if raw_data then
423
+ redis.call('hset', QlessJob.ns .. self.jid, 'data', raw_data)
423
424
  end
424
425
 
425
426
  local queue_obj = Qless.queue(queue)
@@ -454,7 +455,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
454
455
  if redis.call('zscore', 'ql:queues', nextq) == false then
455
456
  redis.call('zadd', 'ql:queues', now, nextq)
456
457
  end
457
-
458
+
458
459
  redis.call('hmset', QlessJob.ns .. self.jid,
459
460
  'state', 'waiting',
460
461
  'worker', '',
@@ -462,7 +463,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
462
463
  'queue', nextq,
463
464
  'expires', 0,
464
465
  'remaining', tonumber(retries))
465
-
466
+
466
467
  if (delay > 0) and (#depends == 0) then
467
468
  queue_obj.scheduled.add(now + delay, self.jid)
468
469
  return 'scheduled'
@@ -505,15 +506,15 @@ function QlessJob:complete(now, worker, queue, data, ...)
505
506
  'queue', '',
506
507
  'expires', 0,
507
508
  'remaining', tonumber(retries))
508
-
509
+
509
510
  local count = Qless.config.get('jobs-history-count')
510
511
  local time = Qless.config.get('jobs-history')
511
-
512
+
512
513
  count = tonumber(count or 50000)
513
514
  time = tonumber(time or 7 * 24 * 60 * 60)
514
-
515
+
515
516
  redis.call('zadd', 'ql:completed', now, self.jid)
516
-
517
+
517
518
  local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
518
519
  for index, jid in ipairs(jids) do
519
520
  local tags = cjson.decode(
@@ -526,7 +527,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
526
527
  redis.call('del', QlessJob.ns .. jid .. '-history')
527
528
  end
528
529
  redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
529
-
530
+
530
531
  jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
531
532
  for index, jid in ipairs(jids) do
532
533
  local tags = cjson.decode(
@@ -539,7 +540,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
539
540
  redis.call('del', QlessJob.ns .. jid .. '-history')
540
541
  end
541
542
  redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
542
-
543
+
543
544
  for i, j in ipairs(redis.call(
544
545
  'smembers', QlessJob.ns .. self.jid .. '-dependents')) do
545
546
  redis.call('srem', QlessJob.ns .. j .. '-dependencies', self.jid)
@@ -561,9 +562,9 @@ function QlessJob:complete(now, worker, queue, data, ...)
561
562
  end
562
563
  end
563
564
  end
564
-
565
+
565
566
  redis.call('del', QlessJob.ns .. self.jid .. '-dependents')
566
-
567
+
567
568
  return 'complete'
568
569
  end
569
570
  end
@@ -583,11 +584,12 @@ function QlessJob:fail(now, worker, group, message, data)
583
584
  'hmget', QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
584
585
 
585
586
  if not state then
586
- error('Fail(): Job does not exist')
587
+ error('Fail(): Job ' .. self.jid .. 'does not exist')
587
588
  elseif state ~= 'running' then
588
- error('Fail(): Job not currently running: ' .. state)
589
+ error('Fail(): Job ' .. self.jid .. 'not currently running: ' .. state)
589
590
  elseif worker ~= oldworker then
590
- error('Fail(): Job running with another worker: ' .. oldworker)
591
+ error('Fail(): Job ' .. self.jid .. ' running with another worker: ' ..
592
+ oldworker)
591
593
  end
592
594
 
593
595
  Qless.publish('log', cjson.encode({
@@ -641,17 +643,19 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
641
643
  assert(worker, 'Retry(): Arg "worker" missing')
642
644
  delay = assert(tonumber(delay or 0),
643
645
  'Retry(): Arg "delay" not a number: ' .. tostring(delay))
644
-
646
+
645
647
  local oldqueue, state, retries, oldworker, priority, failure = unpack(
646
648
  redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state',
647
649
  'retries', 'worker', 'priority', 'failure'))
648
650
 
649
651
  if oldworker == false then
650
- error('Retry(): Job does not exist')
652
+ error('Retry(): Job ' .. self.jid .. ' does not exist')
651
653
  elseif state ~= 'running' then
652
- error('Retry(): Job is not currently running: ' .. state)
654
+ error('Retry(): Job ' .. self.jid .. ' is not currently running: ' ..
655
+ state)
653
656
  elseif oldworker ~= worker then
654
- error('Retry(): Job has been given to another worker: ' .. oldworker)
657
+ error('Retry(): Job ' .. self.jid ..
658
+ ' has been given to another worker: ' .. oldworker)
655
659
  end
656
660
 
657
661
  local remaining = tonumber(redis.call(
@@ -665,7 +669,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
665
669
  if remaining < 0 then
666
670
  local group = group or 'failed-retries-' .. queue
667
671
  self:history(now, 'failed', {['group'] = group})
668
-
672
+
669
673
  redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed',
670
674
  'worker', '',
671
675
  'expires', '')
@@ -688,7 +692,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
688
692
  ['worker'] = unpack(self:data('worker'))
689
693
  }))
690
694
  end
691
-
695
+
692
696
  redis.call('sadd', 'ql:failures', group)
693
697
  redis.call('lpush', 'ql:f:' .. group, self.jid)
694
698
  local bin = now - (now % 86400)
@@ -793,11 +797,14 @@ function QlessJob:heartbeat(now, worker, data)
793
797
  local job_worker, state = unpack(
794
798
  redis.call('hmget', QlessJob.ns .. self.jid, 'worker', 'state'))
795
799
  if job_worker == false then
796
- error('Heartbeat(): Job does not exist')
800
+ error('Heartbeat(): Job ' .. self.jid .. ' does not exist')
797
801
  elseif state ~= 'running' then
798
- error('Heartbeat(): Job not currently running: ' .. state)
802
+ error(
803
+ 'Heartbeat(): Job ' .. self.jid .. ' not currently running: ' .. state)
799
804
  elseif job_worker ~= worker or #job_worker == 0 then
800
- error('Heartbeat(): Job given out to another worker: ' .. job_worker)
805
+ error(
806
+ 'Heartbeat(): Job ' .. self.jid ..
807
+ ' given out to another worker: ' .. job_worker)
801
808
  else
802
809
  if data then
803
810
  redis.call('hmset', QlessJob.ns .. self.jid, 'expires',
@@ -806,9 +813,9 @@ function QlessJob:heartbeat(now, worker, data)
806
813
  redis.call('hmset', QlessJob.ns .. self.jid,
807
814
  'expires', expires, 'worker', worker)
808
815
  end
809
-
816
+
810
817
  redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
811
-
818
+
812
819
  local queue = Qless.queue(
813
820
  redis.call('hget', QlessJob.ns .. self.jid, 'queue'))
814
821
  queue.locks.add(expires, self.jid)
@@ -823,7 +830,7 @@ function QlessJob:priority(priority)
823
830
 
824
831
  local queue = redis.call('hget', QlessJob.ns .. self.jid, 'queue')
825
832
 
826
- if queue == nil then
833
+ if queue == nil or queue == false then
827
834
  error('Priority(): Job ' .. self.jid .. ' does not exist')
828
835
  elseif queue == '' then
829
836
  redis.call('hset', QlessJob.ns .. self.jid, 'priority', priority)
@@ -850,15 +857,15 @@ end
850
857
  function QlessJob:timeout(now)
851
858
  local queue_name, state, worker = unpack(redis.call('hmget',
852
859
  QlessJob.ns .. self.jid, 'queue', 'state', 'worker'))
853
- if queue_name == nil then
854
- error('Timeout(): Job does not exist')
860
+ if queue_name == nil or queue_name == false then
861
+ error('Timeout(): Job ' .. self.jid .. ' does not exist')
855
862
  elseif state ~= 'running' then
856
863
  error('Timeout(): Job ' .. self.jid .. ' not running')
857
864
  else
858
865
  self:history(now, 'timed-out')
859
866
  local queue = Qless.queue(queue_name)
860
867
  queue.locks.remove(self.jid)
861
- queue.work.add(now, math.huge, self.jid)
868
+ queue.work.add(now, '+inf', self.jid)
862
869
  redis.call('hmset', QlessJob.ns .. self.jid,
863
870
  'state', 'stalled', 'expires', 0)
864
871
  local encoded = cjson.encode({
@@ -921,7 +928,7 @@ function QlessJob:history(now, what, item)
921
928
  if count > 0 then
922
929
  local obj = redis.call('lpop', QlessJob.ns .. self.jid .. '-history')
923
930
  redis.call('ltrim', QlessJob.ns .. self.jid .. '-history', -count + 2, -1)
924
- if obj ~= nil then
931
+ if obj ~= nil and obj ~= false then
925
932
  redis.call('lpush', QlessJob.ns .. self.jid .. '-history', obj)
926
933
  end
927
934
  end
@@ -951,8 +958,11 @@ function Qless.queue(name)
951
958
  return redis.call('zrem', queue:prefix('work'), unpack(arg))
952
959
  end
953
960
  end, add = function(now, priority, jid)
961
+ if priority ~= '+inf' then
962
+ priority = priority - (now / 10000000000)
963
+ end
954
964
  return redis.call('zadd',
955
- queue:prefix('work'), priority - (now / 10000000000), jid)
965
+ queue:prefix('work'), priority, jid)
956
966
  end, score = function(jid)
957
967
  return redis.call('zscore', queue:prefix('work'), jid)
958
968
  end, length = function()
@@ -963,10 +973,10 @@ function Qless.queue(name)
963
973
  queue.locks = {
964
974
  expired = function(now, offset, count)
965
975
  return redis.call('zrangebyscore',
966
- queue:prefix('locks'), -math.huge, now, 'LIMIT', offset, count)
976
+ queue:prefix('locks'), '-inf', now, 'LIMIT', offset, count)
967
977
  end, peek = function(now, offset, count)
968
978
  return redis.call('zrangebyscore', queue:prefix('locks'),
969
- now, math.huge, 'LIMIT', offset, count)
979
+ now, '+inf', 'LIMIT', offset, count)
970
980
  end, add = function(expires, jid)
971
981
  redis.call('zadd', queue:prefix('locks'), expires, jid)
972
982
  end, remove = function(...)
@@ -974,7 +984,7 @@ function Qless.queue(name)
974
984
  return redis.call('zrem', queue:prefix('locks'), unpack(arg))
975
985
  end
976
986
  end, running = function(now)
977
- return redis.call('zcount', queue:prefix('locks'), now, math.huge)
987
+ return redis.call('zcount', queue:prefix('locks'), now, '+inf')
978
988
  end, length = function(now)
979
989
  if now then
980
990
  return redis.call('zcount', queue:prefix('locks'), 0, now)
@@ -1065,11 +1075,11 @@ function QlessQueue:stats(now, date)
1065
1075
 
1066
1076
  local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
1067
1077
  local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
1068
-
1078
+
1069
1079
  count = tonumber(count) or 0
1070
1080
  mean = tonumber(mean) or 0
1071
1081
  vk = tonumber(vk)
1072
-
1082
+
1073
1083
  results.count = count or 0
1074
1084
  results.mean = mean or 0
1075
1085
  results.histogram = {}
@@ -1174,17 +1184,17 @@ function QlessQueue:pop(now, worker, count)
1174
1184
  self:stat(now, 'wait', waiting)
1175
1185
  redis.call('hset', QlessJob.ns .. jid,
1176
1186
  'time', string.format("%.20f", now))
1177
-
1187
+
1178
1188
  redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
1179
-
1189
+
1180
1190
  job:update({
1181
1191
  worker = worker,
1182
1192
  expires = expires,
1183
1193
  state = 'running'
1184
1194
  })
1185
-
1195
+
1186
1196
  self.locks.add(expires, jid)
1187
-
1197
+
1188
1198
  local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
1189
1199
  if tracked then
1190
1200
  Qless.publish('popped', jid)
@@ -1224,7 +1234,7 @@ function QlessQueue:stat(now, stat, val)
1224
1234
  redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
1225
1235
  else -- days
1226
1236
  redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
1227
- end
1237
+ end
1228
1238
  redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
1229
1239
  end
1230
1240
 
@@ -1266,8 +1276,8 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
1266
1276
 
1267
1277
  local original = redis.call(
1268
1278
  'smembers', QlessJob.ns .. jid .. '-dependencies')
1269
- for _, dep in pairs(original) do
1270
- if new[dep] == nil then
1279
+ for _, dep in pairs(original) do
1280
+ if new[dep] == nil or new[dep] == false then
1271
1281
  redis.call('srem', QlessJob.ns .. dep .. '-dependents' , jid)
1272
1282
  redis.call('srem', QlessJob.ns .. jid .. '-dependencies', dep)
1273
1283
  end
@@ -1421,7 +1431,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1421
1431
  if #arg % 2 == 1 then
1422
1432
  error('Odd number of additional args: ' .. tostring(arg))
1423
1433
  end
1424
-
1434
+
1425
1435
  local options = {}
1426
1436
  for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end
1427
1437
  options.tags = assert(cjson.decode(options.tags or '{}'),
@@ -1443,7 +1453,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1443
1453
  if old_queue then
1444
1454
  Qless.queue(old_queue).recurring.remove(jid)
1445
1455
  end
1446
-
1456
+
1447
1457
  redis.call('hmset', 'ql:r:' .. jid,
1448
1458
  'jid' , jid,
1449
1459
  'klass' , klass,
@@ -1458,11 +1468,11 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
1458
1468
  'retries' , options.retries,
1459
1469
  'backlog' , options.backlog)
1460
1470
  self.recurring.add(now + offset, jid)
1461
-
1471
+
1462
1472
  if redis.call('zscore', 'ql:queues', self.name) == false then
1463
1473
  redis.call('zadd', 'ql:queues', now, self.name)
1464
1474
  end
1465
-
1475
+
1466
1476
  return jid
1467
1477
  else
1468
1478
  error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
@@ -1493,18 +1503,18 @@ function QlessQueue:check_recurring(now, count)
1493
1503
  )
1494
1504
  end
1495
1505
  end
1496
-
1506
+
1497
1507
  while (score <= now) and (moved < count) do
1498
1508
  local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
1499
1509
  moved = moved + 1
1500
1510
 
1501
1511
  local child_jid = jid .. '-' .. count
1502
-
1512
+
1503
1513
  for i, tag in ipairs(_tags) do
1504
1514
  redis.call('zadd', 'ql:t:' .. tag, now, child_jid)
1505
1515
  redis.call('zincrby', 'ql:tags', 1, tag)
1506
1516
  end
1507
-
1517
+
1508
1518
  redis.call('hmset', QlessJob.ns .. child_jid,
1509
1519
  'jid' , child_jid,
1510
1520
  'klass' , klass,
@@ -1520,9 +1530,9 @@ function QlessQueue:check_recurring(now, count)
1520
1530
  'time' , string.format("%.20f", score),
1521
1531
  'spawned_from_jid', jid)
1522
1532
  Qless.job(child_jid):history(score, 'put', {q = self.name})
1523
-
1533
+
1524
1534
  self.work.add(score, priority, child_jid)
1525
-
1535
+
1526
1536
  score = score + interval
1527
1537
  self.recurring.add(score, jid)
1528
1538
  end
@@ -1587,12 +1597,12 @@ function QlessQueue:invalidate_locks(now, count)
1587
1597
 
1588
1598
  local remaining = tonumber(redis.call(
1589
1599
  'hincrby', QlessJob.ns .. jid, 'remaining', -1))
1590
-
1600
+
1591
1601
  if remaining < 0 then
1592
1602
  self.work.remove(jid)
1593
1603
  self.locks.remove(jid)
1594
1604
  self.scheduled.remove(jid)
1595
-
1605
+
1596
1606
  local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
1597
1607
  local job = Qless.job(jid)
1598
1608
  job:history(now, 'failed', {group = group})
@@ -1607,10 +1617,10 @@ function QlessQueue:invalidate_locks(now, count)
1607
1617
  ['when'] = now,
1608
1618
  ['worker'] = unpack(job:data('worker'))
1609
1619
  }))
1610
-
1620
+
1611
1621
  redis.call('sadd', 'ql:failures', group)
1612
1622
  redis.call('lpush', 'ql:f:' .. group, jid)
1613
-
1623
+
1614
1624
  if redis.call('zscore', 'ql:tracked', jid) ~= false then
1615
1625
  Qless.publish('failed', jid)
1616
1626
  end
@@ -1669,11 +1679,11 @@ function QlessRecurringJob:data()
1669
1679
  local job = redis.call(
1670
1680
  'hmget', 'ql:r:' .. self.jid, 'jid', 'klass', 'state', 'queue',
1671
1681
  'priority', 'interval', 'retries', 'count', 'data', 'tags', 'backlog')
1672
-
1682
+
1673
1683
  if not job[1] then
1674
1684
  return nil
1675
1685
  end
1676
-
1686
+
1677
1687
  return {
1678
1688
  jid = job[1],
1679
1689
  klass = job[2],
@@ -1739,9 +1749,9 @@ function QlessRecurringJob:tag(...)
1739
1749
  tags = cjson.decode(tags)
1740
1750
  local _tags = {}
1741
1751
  for i,v in ipairs(tags) do _tags[v] = true end
1742
-
1743
- for i=1,#arg do if _tags[arg[i]] == nil then table.insert(tags, arg[i]) end end
1744
-
1752
+
1753
+ for i=1,#arg do if _tags[arg[i]] == nil or _tags[arg[i]] == false then table.insert(tags, arg[i]) end end
1754
+
1745
1755
  tags = cjson.encode(tags)
1746
1756
  redis.call('hset', 'ql:r:' .. self.jid, 'tags', tags)
1747
1757
  return tags
@@ -1,5 +1,5 @@
1
1
  # Encoding: utf-8
2
2
 
3
3
  module Qless
4
- VERSION = '0.11.0'
4
+ VERSION = '0.12.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qless
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dan Lecocq
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2017-12-08 00:00:00.000000000 Z
12
+ date: 2018-01-17 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: metriks