qless 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/Gemfile +2 -0
  2. data/README.md +42 -3
  3. data/Rakefile +26 -2
  4. data/{bin → exe}/qless-web +3 -2
  5. data/lib/qless.rb +55 -28
  6. data/lib/qless/config.rb +1 -3
  7. data/lib/qless/job.rb +127 -22
  8. data/lib/qless/job_reservers/round_robin.rb +3 -1
  9. data/lib/qless/job_reservers/shuffled_round_robin.rb +14 -0
  10. data/lib/qless/lua_script.rb +42 -0
  11. data/lib/qless/middleware/redis_reconnect.rb +24 -0
  12. data/lib/qless/middleware/retry_exceptions.rb +43 -0
  13. data/lib/qless/middleware/sentry.rb +70 -0
  14. data/lib/qless/qless-core/cancel.lua +89 -59
  15. data/lib/qless/qless-core/complete.lua +16 -1
  16. data/lib/qless/qless-core/config.lua +12 -0
  17. data/lib/qless/qless-core/deregister_workers.lua +12 -0
  18. data/lib/qless/qless-core/fail.lua +24 -14
  19. data/lib/qless/qless-core/heartbeat.lua +2 -1
  20. data/lib/qless/qless-core/pause.lua +18 -0
  21. data/lib/qless/qless-core/pop.lua +24 -3
  22. data/lib/qless/qless-core/put.lua +14 -1
  23. data/lib/qless/qless-core/qless-lib.lua +2354 -0
  24. data/lib/qless/qless-core/qless.lua +1862 -0
  25. data/lib/qless/qless-core/retry.lua +1 -1
  26. data/lib/qless/qless-core/unfail.lua +54 -0
  27. data/lib/qless/qless-core/unpause.lua +12 -0
  28. data/lib/qless/queue.rb +45 -21
  29. data/lib/qless/server.rb +38 -39
  30. data/lib/qless/server/static/css/docs.css +21 -1
  31. data/lib/qless/server/views/_job.erb +5 -5
  32. data/lib/qless/server/views/overview.erb +14 -9
  33. data/lib/qless/subscriber.rb +48 -0
  34. data/lib/qless/version.rb +1 -1
  35. data/lib/qless/wait_until.rb +19 -0
  36. data/lib/qless/worker.rb +243 -33
  37. metadata +49 -30
  38. data/bin/install_phantomjs +0 -7
  39. data/bin/qless-campfire +0 -106
  40. data/bin/qless-growl +0 -99
  41. data/lib/qless/lua.rb +0 -25
@@ -50,7 +50,7 @@ if remaining < 0 then
50
50
  redis.call('hmset', 'ql:j:' .. jid, 'state', 'failed', 'worker', '',
51
51
  'expires', '', 'history', cjson.encode(history), 'failure', cjson.encode({
52
52
  ['group'] = group,
53
- ['message'] = 'Job exhuasted retries in queue "' .. queue .. '"',
53
+ ['message'] = 'Job exhausted retries in queue "' .. queue .. '"',
54
54
  ['when'] = now,
55
55
  ['worker'] = worker
56
56
  }))
@@ -0,0 +1,54 @@
1
+ -- Unfail(0, now, group, queue, [count])
2
+ --
3
+ -- Move `count` jobs out of the failed state and into the provided queue
4
+
5
+ if #KEYS ~= 0 then
6
+ error('Unfail(): Expected 0 KEYS arguments')
7
+ end
8
+
9
+ local now = assert(tonumber(ARGV[1]), 'Unfail(): Arg "now" missing' )
10
+ local group = assert(ARGV[2] , 'Unfail(): Arg "group" missing')
11
+ local queue = assert(ARGV[3] , 'Unfail(): Arg "queue" missing')
12
+ local count = assert(tonumber(ARGV[4] or 25),
13
+ 'Unfail(): Arg "count" not a number: ' .. tostring(ARGV[4]))
14
+
15
+ -- Get up to that many jobs, and we'll put them in the appropriate queue
16
+ local jids = redis.call('lrange', 'ql:f:' .. group, -count, -1)
17
+
18
+ -- Get each job's original number of retries,
19
+ local jobs = {}
20
+ for index, jid in ipairs(jids) do
21
+ local packed = redis.call('hgetall', 'ql:j:' .. jid)
22
+ local unpacked = {}
23
+ for i = 1, #packed, 2 do unpacked[packed[i]] = packed[i + 1] end
24
+ table.insert(jobs, unpacked)
25
+ end
26
+
27
+ -- And now set each job's state, and put it into the appropriate queue
28
+ local toinsert = {}
29
+ for index, job in ipairs(jobs) do
30
+ job.history = cjson.decode(job.history or '{}')
31
+ table.insert(job.history, {
32
+ q = queue,
33
+ put = math.floor(now)
34
+ })
35
+ redis.call('hmset', 'ql:j:' .. job.jid,
36
+ 'state' , 'waiting',
37
+ 'worker' , '',
38
+ 'expires' , 0,
39
+ 'queue' , queue,
40
+ 'remaining', job.retries or 5,
41
+ 'history' , cjson.encode(job.history))
42
+ table.insert(toinsert, job.priority - (now / 10000000000))
43
+ table.insert(toinsert, job.jid)
44
+ end
45
+
46
+ redis.call('zadd', 'ql:q:' .. queue .. '-work', unpack(toinsert))
47
+
48
+ -- Remove these jobs from the failed state
49
+ redis.call('ltrim', 'ql:f:' .. group, 0, -count - 1)
50
+ if (redis.call('llen', 'ql:f:' .. group) == 0) then
51
+ redis.call('srem', 'ql:failures', group)
52
+ end
53
+
54
+ return #jids
@@ -0,0 +1,12 @@
1
+ -- This script takes the name of the queue(s) and removes it
2
+ -- from the ql:paused_queues set.
3
+ --
4
+ -- Args: The list of queues to pause.
5
+
6
+ if #KEYS > 0 then error('Pause(): No Keys should be provided') end
7
+ if #ARGV < 1 then error('Pause(): Must provide at least one queue to pause') end
8
+
9
+ local key = 'ql:paused_queues'
10
+
11
+ redis.call('srem', key, unpack(ARGV))
12
+
@@ -1,4 +1,3 @@
1
- require "qless/lua"
2
1
  require "qless/job"
3
2
  require "redis"
4
3
  require "json"
@@ -9,54 +8,71 @@ module Qless
9
8
  @name = name
10
9
  @client = client
11
10
  end
12
-
11
+
13
12
  def running(start=0, count=25)
14
13
  @client._jobs.call([], ['running', Time.now.to_f, @name, start, count])
15
14
  end
16
-
15
+
17
16
  def stalled(start=0, count=25)
18
17
  @client._jobs.call([], ['stalled', Time.now.to_f, @name, start, count])
19
18
  end
20
-
19
+
21
20
  def scheduled(start=0, count=25)
22
21
  @client._jobs.call([], ['scheduled', Time.now.to_f, @name, start, count])
23
22
  end
24
-
23
+
25
24
  def depends(start=0, count=25)
26
25
  @client._jobs.call([], ['depends', Time.now.to_f, @name, start, count])
27
26
  end
28
-
27
+
29
28
  def recurring(start=0, count=25)
30
29
  @client._jobs.call([], ['recurring', Time.now.to_f, @name, start, count])
31
30
  end
32
31
  end
33
-
32
+
34
33
  class Queue
35
- attr_reader :name
34
+ attr_reader :name, :client
36
35
  attr_accessor :worker_name
37
-
36
+
38
37
  def initialize(name, client)
39
38
  @client = client
40
39
  @name = name
41
40
  self.worker_name = Qless.worker_name
42
41
  end
43
-
42
+
44
43
  def jobs
45
44
  @jobs ||= QueueJobs.new(@name, @client)
46
45
  end
47
-
46
+
48
47
  def counts
49
48
  JSON.parse(@client._queues.call([], [Time.now.to_i, @name]))
50
49
  end
51
-
50
+
52
51
  def heartbeat
53
- @client.config["#{@name}-heartbeat"]
52
+ get_config :heartbeat
54
53
  end
55
-
54
+
56
55
  def heartbeat=(value)
57
- @client.config["#{@name}-heartbeat"] = value
56
+ set_config :heartbeat, value
57
+ end
58
+
59
+ def max_concurrency
60
+ value = get_config(:"max-concurrency")
61
+ value && Integer(value)
62
+ end
63
+
64
+ def max_concurrency=(value)
65
+ set_config :"max-concurrency", value
58
66
  end
59
-
67
+
68
+ def pause
69
+ @client._pause.call([], [name])
70
+ end
71
+
72
+ def unpause
73
+ @client._unpause.call([], [name])
74
+ end
75
+
60
76
  # Put the described job in this queue
61
77
  # Options include:
62
78
  # => priority (int)
@@ -77,7 +93,7 @@ module Qless
77
93
  'depends', JSON.generate(opts.fetch(:depends, []))
78
94
  ])
79
95
  end
80
-
96
+
81
97
  # Make a recurring job in this queue
82
98
  # Options include:
83
99
  # => priority (int)
@@ -100,23 +116,23 @@ module Qless
100
116
  'retries', opts.fetch(:retries, 5)
101
117
  ])
102
118
  end
103
-
119
+
104
120
  # Pop a work item off the queue
105
121
  def pop(count=nil)
106
122
  results = @client._pop.call([@name], [worker_name, (count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
107
123
  count.nil? ? results[0] : results
108
124
  end
109
-
125
+
110
126
  # Peek at a work item
111
127
  def peek(count=nil)
112
128
  results = @client._peek.call([@name], [(count || 1), Time.now.to_f]).map { |j| Job.new(@client, JSON.parse(j)) }
113
129
  count.nil? ? results[0] : results
114
130
  end
115
-
131
+
116
132
  def stats(date=nil)
117
133
  JSON.parse(@client._stats.call([], [@name, (date || Time.now.to_f)]))
118
134
  end
119
-
135
+
120
136
  # How many items in the queue?
121
137
  def length
122
138
  (@client.redis.multi do
@@ -137,5 +153,13 @@ module Qless
137
153
  return opts unless klass.respond_to?(:default_job_options)
138
154
  klass.default_job_options(data).merge(opts)
139
155
  end
156
+
157
+ def set_config(config, value)
158
+ @client.config["#{@name}-#{config}"] = value
159
+ end
160
+
161
+ def get_config(config)
162
+ @client.config["#{@name}-#{config}"]
163
+ end
140
164
  end
141
165
  end
@@ -16,12 +16,11 @@ module Qless
16
16
  # I'm not sure what this option is -- I'll look it up later
17
17
  # set :static, true
18
18
 
19
- def self.client
20
- @client ||= Qless::Client.new
21
- end
19
+ attr_reader :client
22
20
 
23
- def self.client=(client)
21
+ def initialize(client)
24
22
  @client = client
23
+ super
25
24
  end
26
25
 
27
26
  helpers do
@@ -87,23 +86,23 @@ module Qless
87
86
  end
88
87
 
89
88
  def application_name
90
- return Server.client.config['application']
89
+ return client.config['application']
91
90
  end
92
91
 
93
92
  def queues
94
- return Server.client.queues.counts
93
+ return client.queues.counts
95
94
  end
96
95
 
97
96
  def tracked
98
- return Server.client.jobs.tracked
97
+ return client.jobs.tracked
99
98
  end
100
99
 
101
100
  def workers
102
- return Server.client.workers.counts
101
+ return client.workers.counts
103
102
  end
104
103
 
105
104
  def failed
106
- return Server.client.jobs.failed
105
+ return client.jobs.failed
107
106
  end
108
107
 
109
108
  # Return the supplied object back as JSON
@@ -121,12 +120,12 @@ module Qless
121
120
  # page, then we should probably be caching it
122
121
  def top_tags
123
122
  @top_tags ||= {
124
- :top => Server.client.tags,
123
+ :top => client.tags,
125
124
  :fetched => Time.now
126
125
  }
127
126
  if (Time.now - @top_tags[:fetched]) > 60 then
128
127
  @top_tags = {
129
- :top => Server.client.tags,
128
+ :top => client.tags,
130
129
  :fetched => Time.now
131
130
  }
132
131
  end
@@ -157,7 +156,7 @@ module Qless
157
156
 
158
157
  # Returns a JSON blob with the job counts for various queues
159
158
  get '/queues.json' do
160
- json(Server.client.queues.counts)
159
+ json(client.queues.counts)
161
160
  end
162
161
 
163
162
  get '/queues/?' do
@@ -168,18 +167,18 @@ module Qless
168
167
 
169
168
  # Return the job counts for a specific queue
170
169
  get '/queues/:name.json' do
171
- json(Server.client.queues[params[:name]].counts)
170
+ json(client.queues[params[:name]].counts)
172
171
  end
173
172
 
174
173
  filtered_tabs = %w[ running scheduled stalled depends recurring ].to_set
175
174
  get '/queues/:name/?:tab?' do
176
- queue = Server.client.queues[params[:name]]
175
+ queue = client.queues[params[:name]]
177
176
  tab = params.fetch('tab', 'stats')
178
177
 
179
178
  jobs = if tab == 'waiting'
180
179
  queue.peek(20)
181
180
  elsif filtered_tabs.include?(tab)
182
- paginated(queue.jobs, tab).map { |jid| Server.client.jobs[jid] }
181
+ paginated(queue.jobs, tab).map { |jid| client.jobs[jid] }
183
182
  else
184
183
  []
185
184
  end
@@ -188,13 +187,13 @@ module Qless
188
187
  :title => "Queue #{params[:name]}",
189
188
  :tab => tab,
190
189
  :jobs => jobs,
191
- :queue => Server.client.queues[params[:name]].counts,
190
+ :queue => client.queues[params[:name]].counts,
192
191
  :stats => queue.stats
193
192
  }
194
193
  end
195
194
 
196
195
  get '/failed.json' do
197
- json(Server.client.jobs.failed)
196
+ json(client.jobs.failed)
198
197
  end
199
198
 
200
199
  get '/failed/?' do
@@ -203,7 +202,7 @@ module Qless
203
202
  # should behave or not.
204
203
  erb :failed, :layout => true, :locals => {
205
204
  :title => 'Failed',
206
- :failed => Server.client.jobs.failed.keys.map { |t| Server.client.jobs.failed(t).tap { |f| f['type'] = t } }
205
+ :failed => client.jobs.failed.keys.map { |t| client.jobs.failed(t).tap { |f| f['type'] = t } }
207
206
  }
208
207
  end
209
208
 
@@ -211,7 +210,7 @@ module Qless
211
210
  erb :failed_type, :layout => true, :locals => {
212
211
  :title => 'Failed | ' + params[:type],
213
212
  :type => params[:type],
214
- :failed => paginated(Server.client.jobs, :failed, params[:type])
213
+ :failed => paginated(client.jobs, :failed, params[:type])
215
214
  }
216
215
  end
217
216
 
@@ -225,7 +224,7 @@ module Qless
225
224
  erb :job, :layout => true, :locals => {
226
225
  :title => "Job | #{params[:jid]}",
227
226
  :jid => params[:jid],
228
- :job => Server.client.jobs[params[:jid]]
227
+ :job => client.jobs[params[:jid]]
229
228
  }
230
229
  end
231
230
 
@@ -238,20 +237,20 @@ module Qless
238
237
  get '/workers/:worker' do
239
238
  erb :worker, :layout => true, :locals => {
240
239
  :title => 'Worker | ' + params[:worker],
241
- :worker => Server.client.workers[params[:worker]].tap { |w|
242
- w['jobs'] = w['jobs'].map { |j| Server.client.jobs[j] }
243
- w['stalled'] = w['stalled'].map { |j| Server.client.jobs[j] }
240
+ :worker => client.workers[params[:worker]].tap { |w|
241
+ w['jobs'] = w['jobs'].map { |j| client.jobs[j] }
242
+ w['stalled'] = w['stalled'].map { |j| client.jobs[j] }
244
243
  w['name'] = params[:worker]
245
244
  }
246
245
  }
247
246
  end
248
247
 
249
248
  get '/tag/?' do
250
- jobs = paginated(Server.client.jobs, :tagged, params[:tag])
249
+ jobs = paginated(client.jobs, :tagged, params[:tag])
251
250
  erb :tag, :layout => true, :locals => {
252
251
  :title => "Tag | #{params[:tag]}",
253
252
  :tag => params[:tag],
254
- :jobs => jobs['jobs'].map { |jid| Server.client.jobs[jid] },
253
+ :jobs => jobs['jobs'].map { |jid| client.jobs[jid] },
255
254
  :total => jobs['total']
256
255
  }
257
256
  end
@@ -259,7 +258,7 @@ module Qless
259
258
  get '/config/?' do
260
259
  erb :config, :layout => true, :locals => {
261
260
  :title => 'Config',
262
- :options => Server.client.config.all
261
+ :options => client.config.all
263
262
  }
264
263
  end
265
264
 
@@ -273,7 +272,7 @@ module Qless
273
272
  post "/track/?" do
274
273
  # Expects a JSON-encoded hash with a job id, and optionally some tags
275
274
  data = JSON.parse(request.body.read)
276
- job = Server.client.jobs[data["id"]]
275
+ job = client.jobs[data["id"]]
277
276
  if not job.nil?
278
277
  data.fetch("tags", false) ? job.track(*data["tags"]) : job.track()
279
278
  if request.xhr?
@@ -292,7 +291,7 @@ module Qless
292
291
 
293
292
  post "/untrack/?" do
294
293
  # Expects a JSON-encoded array of job ids to stop tracking
295
- jobs = JSON.parse(request.body.read).map { |jid| Server.client.jobs[jid] }.select { |j| not j.nil? }
294
+ jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }.select { |j| not j.nil? }
296
295
  # Go ahead and cancel all the jobs!
297
296
  jobs.each do |job|
298
297
  job.untrack()
@@ -306,7 +305,7 @@ module Qless
306
305
  r = JSON.parse(request.body.read)
307
306
  r.each_pair do |jid, priority|
308
307
  begin
309
- Server.client.jobs[jid].priority = priority
308
+ client.jobs[jid].priority = priority
310
309
  response[jid] = priority
311
310
  rescue
312
311
  response[jid] = 'failed'
@@ -320,7 +319,7 @@ module Qless
320
319
  response = Hash.new
321
320
  JSON.parse(request.body.read).each_pair do |jid, tags|
322
321
  begin
323
- Server.client.jobs[jid].tag(*tags)
322
+ client.jobs[jid].tag(*tags)
324
323
  response[jid] = tags
325
324
  rescue
326
325
  response[jid] = 'failed'
@@ -334,7 +333,7 @@ module Qless
334
333
  response = Hash.new
335
334
  JSON.parse(request.body.read).each_pair do |jid, tags|
336
335
  begin
337
- Server.client.jobs[jid].untag(*tags)
336
+ client.jobs[jid].untag(*tags)
338
337
  response[jid] = tags
339
338
  rescue
340
339
  response[jid] = 'failed'
@@ -349,7 +348,7 @@ module Qless
349
348
  if data["id"].nil? or data["queue"].nil?
350
349
  halt 400, "Need id and queue arguments"
351
350
  else
352
- job = Server.client.jobs[data["id"]]
351
+ job = client.jobs[data["id"]]
353
352
  if job.nil?
354
353
  halt 404, "Could not find job"
355
354
  else
@@ -365,7 +364,7 @@ module Qless
365
364
  if data["id"].nil?
366
365
  halt 400, "Need id"
367
366
  else
368
- job = Server.client.jobs[data["id"]]
367
+ job = client.jobs[data["id"]]
369
368
  if job.nil?
370
369
  halt 404, "Could not find job"
371
370
  else
@@ -381,11 +380,11 @@ module Qless
381
380
  if data["id"].nil?
382
381
  halt 400, "Need id"
383
382
  else
384
- job = Server.client.jobs[data["id"]]
383
+ job = client.jobs[data["id"]]
385
384
  if job.nil?
386
385
  halt 404, "Could not find job"
387
386
  else
388
- queue = job.history[-1]["q"]
387
+ queue = job.raw_queue_history[-1]["q"]
389
388
  job.move(queue)
390
389
  return json({ :id => data["id"], :queue => queue})
391
390
  end
@@ -399,8 +398,8 @@ module Qless
399
398
  if data["type"].nil?
400
399
  halt 400, "Neet type"
401
400
  else
402
- return json(Server.client.jobs.failed(data["type"], 0, 500)['jobs'].map do |job|
403
- queue = job.history[-1]["q"]
401
+ return json(client.jobs.failed(data["type"], 0, 500)['jobs'].map do |job|
402
+ queue = job.raw_queue_history[-1]["q"]
404
403
  job.move(queue)
405
404
  { :id => job.jid, :queue => queue}
406
405
  end)
@@ -409,7 +408,7 @@ module Qless
409
408
 
410
409
  post "/cancel/?" do
411
410
  # Expects a JSON-encoded array of job ids to cancel
412
- jobs = JSON.parse(request.body.read).map { |jid| Server.client.jobs[jid] }.select { |j| not j.nil? }
411
+ jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }.select { |j| not j.nil? }
413
412
  # Go ahead and cancel all the jobs!
414
413
  jobs.each do |job|
415
414
  job.cancel()
@@ -428,7 +427,7 @@ module Qless
428
427
  if data["type"].nil?
429
428
  halt 400, "Neet type"
430
429
  else
431
- return json(Server.client.jobs.failed(data["type"])['jobs'].map do |job|
430
+ return json(client.jobs.failed(data["type"])['jobs'].map do |job|
432
431
  job.cancel()
433
432
  { :id => job.jid }
434
433
  end)