reqless 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +8 -0
- data/README.md +648 -0
- data/Rakefile +117 -0
- data/bin/docker-build-and-test +22 -0
- data/exe/reqless-web +11 -0
- data/lib/reqless/config.rb +31 -0
- data/lib/reqless/failure_formatter.rb +43 -0
- data/lib/reqless/job.rb +496 -0
- data/lib/reqless/job_reservers/ordered.rb +29 -0
- data/lib/reqless/job_reservers/round_robin.rb +46 -0
- data/lib/reqless/job_reservers/shuffled_round_robin.rb +21 -0
- data/lib/reqless/lua/reqless-lib.lua +2965 -0
- data/lib/reqless/lua/reqless.lua +2545 -0
- data/lib/reqless/lua_script.rb +90 -0
- data/lib/reqless/middleware/requeue_exceptions.rb +94 -0
- data/lib/reqless/middleware/retry_exceptions.rb +72 -0
- data/lib/reqless/middleware/sentry.rb +66 -0
- data/lib/reqless/middleware/timeout.rb +63 -0
- data/lib/reqless/queue.rb +189 -0
- data/lib/reqless/queue_priority_pattern.rb +16 -0
- data/lib/reqless/server/static/css/bootstrap-responsive.css +686 -0
- data/lib/reqless/server/static/css/bootstrap-responsive.min.css +12 -0
- data/lib/reqless/server/static/css/bootstrap.css +3991 -0
- data/lib/reqless/server/static/css/bootstrap.min.css +689 -0
- data/lib/reqless/server/static/css/codemirror.css +112 -0
- data/lib/reqless/server/static/css/docs.css +839 -0
- data/lib/reqless/server/static/css/jquery.noty.css +105 -0
- data/lib/reqless/server/static/css/noty_theme_twitter.css +137 -0
- data/lib/reqless/server/static/css/style.css +200 -0
- data/lib/reqless/server/static/favicon.ico +0 -0
- data/lib/reqless/server/static/img/glyphicons-halflings-white.png +0 -0
- data/lib/reqless/server/static/img/glyphicons-halflings.png +0 -0
- data/lib/reqless/server/static/js/bootstrap-alert.js +94 -0
- data/lib/reqless/server/static/js/bootstrap-scrollspy.js +125 -0
- data/lib/reqless/server/static/js/bootstrap-tab.js +130 -0
- data/lib/reqless/server/static/js/bootstrap-tooltip.js +270 -0
- data/lib/reqless/server/static/js/bootstrap-typeahead.js +285 -0
- data/lib/reqless/server/static/js/bootstrap.js +1726 -0
- data/lib/reqless/server/static/js/bootstrap.min.js +6 -0
- data/lib/reqless/server/static/js/codemirror.js +2972 -0
- data/lib/reqless/server/static/js/jquery.noty.js +220 -0
- data/lib/reqless/server/static/js/mode/javascript.js +360 -0
- data/lib/reqless/server/static/js/theme/cobalt.css +18 -0
- data/lib/reqless/server/static/js/theme/eclipse.css +25 -0
- data/lib/reqless/server/static/js/theme/elegant.css +10 -0
- data/lib/reqless/server/static/js/theme/lesser-dark.css +45 -0
- data/lib/reqless/server/static/js/theme/monokai.css +28 -0
- data/lib/reqless/server/static/js/theme/neat.css +9 -0
- data/lib/reqless/server/static/js/theme/night.css +21 -0
- data/lib/reqless/server/static/js/theme/rubyblue.css +21 -0
- data/lib/reqless/server/static/js/theme/xq-dark.css +46 -0
- data/lib/reqless/server/views/_job.erb +259 -0
- data/lib/reqless/server/views/_job_list.erb +8 -0
- data/lib/reqless/server/views/_pagination.erb +7 -0
- data/lib/reqless/server/views/about.erb +130 -0
- data/lib/reqless/server/views/completed.erb +11 -0
- data/lib/reqless/server/views/config.erb +14 -0
- data/lib/reqless/server/views/failed.erb +48 -0
- data/lib/reqless/server/views/failed_type.erb +18 -0
- data/lib/reqless/server/views/job.erb +17 -0
- data/lib/reqless/server/views/layout.erb +451 -0
- data/lib/reqless/server/views/overview.erb +137 -0
- data/lib/reqless/server/views/queue.erb +125 -0
- data/lib/reqless/server/views/queues.erb +45 -0
- data/lib/reqless/server/views/tag.erb +6 -0
- data/lib/reqless/server/views/throttles.erb +38 -0
- data/lib/reqless/server/views/track.erb +75 -0
- data/lib/reqless/server/views/worker.erb +34 -0
- data/lib/reqless/server/views/workers.erb +14 -0
- data/lib/reqless/server.rb +549 -0
- data/lib/reqless/subscriber.rb +74 -0
- data/lib/reqless/test_helpers/worker_helpers.rb +55 -0
- data/lib/reqless/throttle.rb +57 -0
- data/lib/reqless/version.rb +5 -0
- data/lib/reqless/worker/base.rb +237 -0
- data/lib/reqless/worker/forking.rb +215 -0
- data/lib/reqless/worker/serial.rb +41 -0
- data/lib/reqless/worker.rb +5 -0
- data/lib/reqless.rb +309 -0
- metadata +399 -0
@@ -0,0 +1,549 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'sinatra/base'
|
4
|
+
require 'reqless'
|
5
|
+
|
6
|
+
module Reqless
|
7
|
+
# The Reqless web interface
|
8
|
+
class Server < Sinatra::Base
|
9
|
+
# Path-y-ness
|
10
|
+
dir = File.dirname(File.expand_path(__FILE__))
|
11
|
+
set :views , "#{dir}/server/views"
|
12
|
+
set :public_folder, "#{dir}/server/static"
|
13
|
+
|
14
|
+
# For debugging purposes at least, I want this
|
15
|
+
set :reload_templates, true
|
16
|
+
|
17
|
+
# I'm not sure what this option is -- I'll look it up later
|
18
|
+
# set :static, true
|
19
|
+
|
20
|
+
attr_reader :client
|
21
|
+
|
22
|
+
def initialize(client)
|
23
|
+
@client = client
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
helpers do
|
28
|
+
include Rack::Utils
|
29
|
+
|
30
|
+
def url_path(*path_parts)
|
31
|
+
[path_prefix, path_parts].join('/').squeeze('/')
|
32
|
+
end
|
33
|
+
alias_method :u, :url_path
|
34
|
+
|
35
|
+
def path_prefix
|
36
|
+
request.env['SCRIPT_NAME']
|
37
|
+
end
|
38
|
+
|
39
|
+
def url_with_modified_query
|
40
|
+
url = URI(request.url)
|
41
|
+
existing_query = Rack::Utils.parse_query(url.query)
|
42
|
+
url.query = Rack::Utils.build_query(yield existing_query)
|
43
|
+
url.to_s
|
44
|
+
end
|
45
|
+
|
46
|
+
def page_url(offset)
|
47
|
+
url_with_modified_query do |query|
|
48
|
+
query.merge('page' => current_page + offset)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def next_page_url
|
53
|
+
page_url(1)
|
54
|
+
end
|
55
|
+
|
56
|
+
def prev_page_url
|
57
|
+
page_url(-1)
|
58
|
+
end
|
59
|
+
|
60
|
+
def current_page
|
61
|
+
@current_page ||= begin
|
62
|
+
Integer(params[:page])
|
63
|
+
rescue
|
64
|
+
1
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
PAGE_SIZE = 25
|
69
|
+
def pagination_values
|
70
|
+
start = (current_page - 1) * PAGE_SIZE
|
71
|
+
[start, PAGE_SIZE]
|
72
|
+
end
|
73
|
+
|
74
|
+
def paginated(reqless_object, method, *args)
|
75
|
+
reqless_object.send(method, *(args + pagination_values))
|
76
|
+
end
|
77
|
+
|
78
|
+
def tabs
|
79
|
+
[
|
80
|
+
{ name: 'Queues' , path: '/queues' },
|
81
|
+
{ name: 'Throttles', path: '/throttles'},
|
82
|
+
{ name: 'Workers' , path: '/workers' },
|
83
|
+
{ name: 'Track' , path: '/track' },
|
84
|
+
{ name: 'Failed' , path: '/failed' },
|
85
|
+
{ name: 'Completed', path: '/completed'},
|
86
|
+
{ name: 'Config' , path: '/config' },
|
87
|
+
{ name: 'About' , path: '/about' }
|
88
|
+
]
|
89
|
+
end
|
90
|
+
|
91
|
+
def application_name
|
92
|
+
client.config['application']
|
93
|
+
end
|
94
|
+
|
95
|
+
def queues
|
96
|
+
client.queues.counts
|
97
|
+
end
|
98
|
+
|
99
|
+
def throttles
|
100
|
+
client.throttles.counts
|
101
|
+
end
|
102
|
+
|
103
|
+
def tracked
|
104
|
+
client.jobs.tracked
|
105
|
+
end
|
106
|
+
|
107
|
+
def workers
|
108
|
+
client.workers.counts
|
109
|
+
end
|
110
|
+
|
111
|
+
def failed
|
112
|
+
client.jobs.failed
|
113
|
+
end
|
114
|
+
|
115
|
+
# Return the supplied object back as JSON
|
116
|
+
def json(obj)
|
117
|
+
content_type :json
|
118
|
+
obj.to_json
|
119
|
+
end
|
120
|
+
|
121
|
+
# Make the id acceptable as an id / att in HTML
|
122
|
+
def sanitize_attr(attr)
|
123
|
+
attr.gsub(/[^a-zA-Z\:\_]/, '-')
|
124
|
+
end
|
125
|
+
|
126
|
+
# What are the top tags? Since it might go on, say, every
|
127
|
+
# page, then we should probably be caching it
|
128
|
+
def top_tags
|
129
|
+
@top_tags ||= {
|
130
|
+
top: client.tags,
|
131
|
+
fetched: Time.now
|
132
|
+
}
|
133
|
+
if (Time.now - @top_tags[:fetched]) > 60
|
134
|
+
@top_tags = {
|
135
|
+
top: client.tags,
|
136
|
+
fetched: Time.now
|
137
|
+
}
|
138
|
+
end
|
139
|
+
@top_tags[:top]
|
140
|
+
end
|
141
|
+
|
142
|
+
def strftime(t)
|
143
|
+
# From http://stackoverflow.com/questions/195740
|
144
|
+
diff_seconds = Time.now - t
|
145
|
+
formatted = t.strftime('%b %e, %Y %H:%M:%S')
|
146
|
+
case diff_seconds
|
147
|
+
when 0 .. 59
|
148
|
+
"#{formatted} (#{diff_seconds.to_i} seconds ago)"
|
149
|
+
when 60 ... 3600
|
150
|
+
"#{formatted} (#{(diff_seconds / 60).to_i} minutes ago)"
|
151
|
+
when 3600 ... 3600 * 24
|
152
|
+
"#{formatted} (#{(diff_seconds / 3600).to_i} hours ago)"
|
153
|
+
when (3600 * 24) ... (3600 * 24 * 30)
|
154
|
+
"#{formatted} (#{(diff_seconds / (3600 * 24)).to_i} days ago)"
|
155
|
+
else
|
156
|
+
formatted
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
get '/?' do
|
162
|
+
erb :overview, layout: true, locals: { title: 'Overview' }
|
163
|
+
end
|
164
|
+
|
165
|
+
# Returns a JSON blob with the job counts for various queues
|
166
|
+
get '/queues.json' do
|
167
|
+
json(client.queues.counts)
|
168
|
+
end
|
169
|
+
|
170
|
+
get '/queues/?' do
|
171
|
+
erb :queues, layout: true, locals: {
|
172
|
+
title: 'Queues'
|
173
|
+
}
|
174
|
+
end
|
175
|
+
|
176
|
+
# Return the job counts for a specific queue
|
177
|
+
get '/queues/:name.json' do
|
178
|
+
json(client.queues[params[:name]].counts)
|
179
|
+
end
|
180
|
+
|
181
|
+
filtered_tabs = %w[ running throttled scheduled stalled depends recurring ].to_set
|
182
|
+
get '/queues/:name/?:tab?' do
|
183
|
+
queue = client.queues[params[:name]]
|
184
|
+
tab = params.fetch('tab', 'stats')
|
185
|
+
|
186
|
+
jobs = []
|
187
|
+
if tab == 'waiting'
|
188
|
+
jobs = queue.peek(*pagination_values)
|
189
|
+
elsif filtered_tabs.include?(tab)
|
190
|
+
jobs = paginated(queue.jobs, tab).map { |jid| client.jobs[jid] }
|
191
|
+
end
|
192
|
+
|
193
|
+
erb :queue, layout: true, locals: {
|
194
|
+
title: "Queue #{params[:name]}",
|
195
|
+
tab: tab,
|
196
|
+
jobs: jobs,
|
197
|
+
queue: client.queues[params[:name]].counts,
|
198
|
+
stats: queue.stats
|
199
|
+
}
|
200
|
+
end
|
201
|
+
|
202
|
+
get '/throttles/?' do
|
203
|
+
erb :throttles, layout: true, locals: {
|
204
|
+
title: 'Throttles'
|
205
|
+
}
|
206
|
+
end
|
207
|
+
|
208
|
+
post '/throttle' do
|
209
|
+
request.body.rewind
|
210
|
+
# Expects a JSON object: {'id': id, 'maximum': maximum}
|
211
|
+
data = JSON.parse(request.body.read)
|
212
|
+
if data['id'].nil? || data['maximum'].nil?
|
213
|
+
halt 400, 'Need throttle id and maximum value'
|
214
|
+
else
|
215
|
+
throttle = Throttle.new(data['id'], client)
|
216
|
+
throttle.maximum = data['maximum']
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
put '/throttle' do
|
221
|
+
request.body.rewind
|
222
|
+
# Expects a JSON object: {'id': id, 'expiration': expiration}
|
223
|
+
data = JSON.parse(request.body.read)
|
224
|
+
if data['id'].nil? || data['expiration'].nil?
|
225
|
+
halt 400, 'Need throttle id and expiration value'
|
226
|
+
else
|
227
|
+
throttle = Throttle.new(data['id'], client)
|
228
|
+
throttle.expiration = data['expiration']
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
delete '/throttle' do
|
233
|
+
request.body.rewind
|
234
|
+
# Expects a JSON object: {'id': id}
|
235
|
+
data = JSON.parse(request.body.read)
|
236
|
+
if data['id'].nil?
|
237
|
+
halt 400, 'Need throttle id'
|
238
|
+
else
|
239
|
+
throttle = Throttle.new(data['id'], client)
|
240
|
+
throttle.delete
|
241
|
+
return json({id: throttle.id, maximum: throttle.maximum})
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
get '/failed.json' do
|
246
|
+
json(client.jobs.failed)
|
247
|
+
end
|
248
|
+
|
249
|
+
get '/failed/?' do
|
250
|
+
# reqless-core doesn't provide functionality this way, so we'll
|
251
|
+
# do it ourselves. I'm not sure if this is how the core library
|
252
|
+
# should behave or not.
|
253
|
+
erb :failed, layout: true, locals: {
|
254
|
+
title: 'Failed',
|
255
|
+
failed: client.jobs.failed.keys.map do |t|
|
256
|
+
client.jobs.failed(t).tap { |f| f['type'] = t }
|
257
|
+
end
|
258
|
+
}
|
259
|
+
end
|
260
|
+
|
261
|
+
get '/failed/:type/?' do
|
262
|
+
erb :failed_type, layout: true, locals: {
|
263
|
+
title: 'Failed | ' + params[:type],
|
264
|
+
type: params[:type],
|
265
|
+
failed: paginated(client.jobs, :failed, params[:type])
|
266
|
+
}
|
267
|
+
end
|
268
|
+
|
269
|
+
get '/completed/?' do
|
270
|
+
completed = paginated(client.jobs, :complete)
|
271
|
+
erb :completed, layout: true, locals: {
|
272
|
+
title: 'Completed',
|
273
|
+
jobs: completed.map { |jid| client.jobs[jid] }
|
274
|
+
}
|
275
|
+
end
|
276
|
+
|
277
|
+
get '/track/?' do
|
278
|
+
erb :track, layout: true, locals: {
|
279
|
+
title: 'Track'
|
280
|
+
}
|
281
|
+
end
|
282
|
+
|
283
|
+
get '/jobs/:jid' do
|
284
|
+
erb :job, layout: true, locals: {
|
285
|
+
title: "Job | #{params[:jid]}",
|
286
|
+
jid: params[:jid],
|
287
|
+
job: client.jobs[params[:jid]]
|
288
|
+
}
|
289
|
+
end
|
290
|
+
|
291
|
+
get '/workers/?' do
|
292
|
+
erb :workers, layout: true, locals: {
|
293
|
+
title: 'Workers'
|
294
|
+
}
|
295
|
+
end
|
296
|
+
|
297
|
+
get '/workers/:worker' do
|
298
|
+
erb :worker, layout: true, locals: {
|
299
|
+
title: 'Worker | ' + params[:worker],
|
300
|
+
worker: client.workers[params[:worker]].tap do |w|
|
301
|
+
w['jobs'] = w['jobs'].map { |j| client.jobs[j] }
|
302
|
+
w['stalled'] = w['stalled'].map { |j| client.jobs[j] }
|
303
|
+
w['name'] = params[:worker]
|
304
|
+
end
|
305
|
+
}
|
306
|
+
end
|
307
|
+
|
308
|
+
get '/tag/?' do
|
309
|
+
jobs = paginated(client.jobs, :tagged, params[:tag])
|
310
|
+
erb :tag, layout: true, locals: {
|
311
|
+
title: "Tag | #{params[:tag]}",
|
312
|
+
tag: params[:tag],
|
313
|
+
jobs: jobs['jobs'].map { |jid| client.jobs[jid] },
|
314
|
+
total: jobs['total']
|
315
|
+
}
|
316
|
+
end
|
317
|
+
|
318
|
+
get '/config/?' do
|
319
|
+
erb :config, layout: true, locals: {
|
320
|
+
title: 'Config',
|
321
|
+
options: client.config.all
|
322
|
+
}
|
323
|
+
end
|
324
|
+
|
325
|
+
get '/about/?' do
|
326
|
+
erb :about, layout: true, locals: {
|
327
|
+
title: 'About'
|
328
|
+
}
|
329
|
+
end
|
330
|
+
|
331
|
+
# These are the bits where we accept AJAX requests
|
332
|
+
post '/track/?' do
|
333
|
+
request.body.rewind
|
334
|
+
# Expects a JSON-encoded hash with a job id, and optionally some tags
|
335
|
+
data = JSON.parse(request.body.read)
|
336
|
+
job = client.jobs[data['id']]
|
337
|
+
if !job.nil?
|
338
|
+
data.fetch('tags', false) ? job.track(*data['tags']) : job.track
|
339
|
+
if request.xhr?
|
340
|
+
json({ tracked: [job.jid] })
|
341
|
+
else
|
342
|
+
redirect to('/track')
|
343
|
+
end
|
344
|
+
else
|
345
|
+
if request.xhr?
|
346
|
+
json({ tracked: [] })
|
347
|
+
else
|
348
|
+
redirect to(request.referrer)
|
349
|
+
end
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
post '/untrack/?' do
|
354
|
+
request.body.rewind
|
355
|
+
# Expects a JSON-encoded array of job ids to stop tracking
|
356
|
+
jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
|
357
|
+
jobs.compact!
|
358
|
+
# Go ahead and cancel all the jobs!
|
359
|
+
jobs.each do |job|
|
360
|
+
job.untrack
|
361
|
+
end
|
362
|
+
return json({ untracked: jobs.map { |job| job.jid } })
|
363
|
+
end
|
364
|
+
|
365
|
+
post '/priority/?' do
|
366
|
+
request.body.rewind
|
367
|
+
# Expects a JSON-encoded dictionary of jid => priority
|
368
|
+
response = Hash.new
|
369
|
+
r = JSON.parse(request.body.read)
|
370
|
+
r.each_pair do |jid, priority|
|
371
|
+
begin
|
372
|
+
client.jobs[jid].priority = priority
|
373
|
+
response[jid] = priority
|
374
|
+
rescue
|
375
|
+
response[jid] = 'failed'
|
376
|
+
end
|
377
|
+
end
|
378
|
+
return json(response)
|
379
|
+
end
|
380
|
+
|
381
|
+
post '/pause/?' do
|
382
|
+
request.body.rewind
|
383
|
+
# Expects JSON blob: {'queue': <queue>}
|
384
|
+
r = JSON.parse(request.body.read)
|
385
|
+
if r['queue']
|
386
|
+
@client.queues[r['queue']].pause
|
387
|
+
return json({ queue: 'paused' })
|
388
|
+
else
|
389
|
+
raise 'No queue provided'
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
post '/unpause/?' do
|
394
|
+
request.body.rewind
|
395
|
+
# Expects JSON blob: {'queue': <queue>}
|
396
|
+
r = JSON.parse(request.body.read)
|
397
|
+
if r['queue']
|
398
|
+
@client.queues[r['queue']].unpause
|
399
|
+
return json({ queue: 'unpaused' })
|
400
|
+
else
|
401
|
+
raise 'No queue provided'
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
post '/timeout/?' do
|
406
|
+
request.body.rewind
|
407
|
+
# Expects JSON blob: {'jid': <jid>}
|
408
|
+
r = JSON.parse(request.body.read)
|
409
|
+
if r['jid']
|
410
|
+
@client.jobs[r['jid']].timeout
|
411
|
+
return json({ jid: r['jid'] })
|
412
|
+
else
|
413
|
+
raise 'No jid provided'
|
414
|
+
end
|
415
|
+
end
|
416
|
+
|
417
|
+
post '/tag/?' do
|
418
|
+
request.body.rewind
|
419
|
+
# Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
|
420
|
+
response = Hash.new
|
421
|
+
JSON.parse(request.body.read).each_pair do |jid, tags|
|
422
|
+
begin
|
423
|
+
client.jobs[jid].tag(*tags)
|
424
|
+
response[jid] = tags
|
425
|
+
rescue
|
426
|
+
response[jid] = 'failed'
|
427
|
+
end
|
428
|
+
end
|
429
|
+
return json(response)
|
430
|
+
end
|
431
|
+
|
432
|
+
post '/untag/?' do
|
433
|
+
request.body.rewind
|
434
|
+
# Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
|
435
|
+
response = Hash.new
|
436
|
+
JSON.parse(request.body.read).each_pair do |jid, tags|
|
437
|
+
begin
|
438
|
+
client.jobs[jid].untag(*tags)
|
439
|
+
response[jid] = tags
|
440
|
+
rescue
|
441
|
+
response[jid] = 'failed'
|
442
|
+
end
|
443
|
+
end
|
444
|
+
return json(response)
|
445
|
+
end
|
446
|
+
|
447
|
+
post '/move/?' do
|
448
|
+
request.body.rewind
|
449
|
+
# Expects a JSON-encoded hash of id: jid, and queue: queue_name
|
450
|
+
data = JSON.parse(request.body.read)
|
451
|
+
if data['id'].nil? || data['queue'].nil?
|
452
|
+
halt 400, 'Need id and queue arguments'
|
453
|
+
else
|
454
|
+
job = client.jobs[data['id']]
|
455
|
+
if job.nil?
|
456
|
+
halt 404, 'Could not find job'
|
457
|
+
else
|
458
|
+
job.requeue(data['queue'])
|
459
|
+
return json({ id: data['id'], queue: data['queue'] })
|
460
|
+
end
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
464
|
+
post '/undepend/?' do
|
465
|
+
request.body.rewind
|
466
|
+
# Expects a JSON-encoded hash of id: jid, and queue: queue_name
|
467
|
+
data = JSON.parse(request.body.read)
|
468
|
+
if data['id'].nil?
|
469
|
+
halt 400, 'Need id'
|
470
|
+
else
|
471
|
+
job = client.jobs[data['id']]
|
472
|
+
if job.nil?
|
473
|
+
halt 404, 'Could not find job'
|
474
|
+
else
|
475
|
+
job.undepend(data['dependency'])
|
476
|
+
return json({ id: data['id'] })
|
477
|
+
end
|
478
|
+
end
|
479
|
+
end
|
480
|
+
|
481
|
+
post '/retry/?' do
|
482
|
+
request.body.rewind
|
483
|
+
# Expects a JSON-encoded hash of id: jid, and queue: queue_name
|
484
|
+
data = JSON.parse(request.body.read)
|
485
|
+
if data['id'].nil?
|
486
|
+
halt 400, 'Need id'
|
487
|
+
else
|
488
|
+
job = client.jobs[data['id']]
|
489
|
+
if job.nil?
|
490
|
+
halt 404, 'Could not find job'
|
491
|
+
else
|
492
|
+
job.requeue(job.queue_name)
|
493
|
+
return json({ id: data['id'], queue: job.queue_name })
|
494
|
+
end
|
495
|
+
end
|
496
|
+
end
|
497
|
+
|
498
|
+
# Retry all the failures of a particular type
|
499
|
+
post '/retryall/?' do
|
500
|
+
request.body.rewind
|
501
|
+
# Expects a JSON-encoded hash of type: failure-type
|
502
|
+
data = JSON.parse(request.body.read)
|
503
|
+
if data['type'].nil?
|
504
|
+
halt 400, 'Neet type'
|
505
|
+
else
|
506
|
+
jobs = client.jobs.failed(data['type'], 0, 500)['jobs']
|
507
|
+
results = jobs.map do |job|
|
508
|
+
job.requeue(job.queue_name)
|
509
|
+
{ id: job.jid, queue: job.queue_name }
|
510
|
+
end
|
511
|
+
return json(results)
|
512
|
+
end
|
513
|
+
end
|
514
|
+
|
515
|
+
post '/cancel/?' do
|
516
|
+
request.body.rewind
|
517
|
+
# Expects a JSON-encoded array of job ids to cancel
|
518
|
+
jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
|
519
|
+
jobs.compact!
|
520
|
+
# Go ahead and cancel all the jobs!
|
521
|
+
jobs.each do |job|
|
522
|
+
job.cancel
|
523
|
+
end
|
524
|
+
|
525
|
+
if request.xhr?
|
526
|
+
return json({ canceled: jobs.map { |job| job.jid } })
|
527
|
+
else
|
528
|
+
redirect to(request.referrer)
|
529
|
+
end
|
530
|
+
end
|
531
|
+
|
532
|
+
post '/cancelall/?' do
|
533
|
+
request.body.rewind
|
534
|
+
# Expects a JSON-encoded hash of type: failure-type
|
535
|
+
data = JSON.parse(request.body.read)
|
536
|
+
if data['type'].nil?
|
537
|
+
halt 400, 'Neet type'
|
538
|
+
else
|
539
|
+
return json(client.jobs.failed(data['type'])['jobs'].map do |job|
|
540
|
+
job.cancel
|
541
|
+
{ id: job.jid }
|
542
|
+
end)
|
543
|
+
end
|
544
|
+
end
|
545
|
+
|
546
|
+
# start the server if ruby file executed directly
|
547
|
+
run! if app_file == $PROGRAM_NAME
|
548
|
+
end
|
549
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'logger'
|
4
|
+
require 'thread'
|
5
|
+
|
6
|
+
module Reqless
|
7
|
+
# A class used for subscribing to messages in a thread
|
8
|
+
class Subscriber
|
9
|
+
def self.start(*args, &block)
|
10
|
+
new(*args, &block).tap(&:start)
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :channel, :redis
|
14
|
+
|
15
|
+
def initialize(client, channel, options = {}, &message_received_callback)
|
16
|
+
@channel = channel
|
17
|
+
@message_received_callback = message_received_callback
|
18
|
+
@log = options.fetch(:log) { ::Logger.new($stderr) }
|
19
|
+
|
20
|
+
# pub/sub blocks the connection so we must use a different redis
|
21
|
+
# connection
|
22
|
+
@client_redis = client.redis
|
23
|
+
@listener_redis = client.new_redis_connection
|
24
|
+
|
25
|
+
@my_channel = Reqless.generate_jid
|
26
|
+
end
|
27
|
+
|
28
|
+
# Start a thread listening
|
29
|
+
def start
|
30
|
+
queue = ::Queue.new
|
31
|
+
|
32
|
+
@thread = Thread.start do
|
33
|
+
begin
|
34
|
+
@listener_redis.subscribe(@channel, @my_channel) do |on|
|
35
|
+
on.subscribe do |channel|
|
36
|
+
# insert nil into the queue to indicate we've
|
37
|
+
# successfully subscribed
|
38
|
+
queue << nil if channel == @channel
|
39
|
+
end
|
40
|
+
|
41
|
+
on.message do |channel, message|
|
42
|
+
handle_message(channel, message)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
# Watch for any exceptions so we don't block forever if
|
46
|
+
# subscribing to the channel fails
|
47
|
+
rescue Exception => e
|
48
|
+
queue << e
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
if (exception = queue.pop)
|
53
|
+
raise exception
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def stop
|
58
|
+
@client_redis.publish(@my_channel, 'disconnect')
|
59
|
+
@thread.join
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def handle_message(channel, message)
|
65
|
+
if channel == @my_channel
|
66
|
+
@listener_redis.unsubscribe(@channel, @my_channel) if message == "disconnect"
|
67
|
+
else
|
68
|
+
@message_received_callback.call(self, JSON.parse(message))
|
69
|
+
end
|
70
|
+
rescue Exception => error
|
71
|
+
@log.error("Reqless::Subscriber") { error }
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Reqless
|
2
|
+
module WorkerHelpers
|
3
|
+
# Yield with a worker running, and then clean the worker up afterwards
|
4
|
+
def run_worker_concurrently_with(worker, &block)
|
5
|
+
thread = Thread.start { stop_worker_after(worker, &block) }
|
6
|
+
thread.abort_on_exception = true
|
7
|
+
worker.run
|
8
|
+
ensure
|
9
|
+
thread.join(0.1)
|
10
|
+
end
|
11
|
+
|
12
|
+
def stop_worker_after(worker, &block)
|
13
|
+
yield
|
14
|
+
ensure
|
15
|
+
worker.stop!
|
16
|
+
end
|
17
|
+
|
18
|
+
# Run only the given number of jobs, then stop
|
19
|
+
def run_jobs(worker, count)
|
20
|
+
worker.extend Module.new {
|
21
|
+
define_method(:jobs) do
|
22
|
+
base_enum = super()
|
23
|
+
Enumerator.new do |enum|
|
24
|
+
count.times { enum << base_enum.next }
|
25
|
+
end
|
26
|
+
end
|
27
|
+
}
|
28
|
+
|
29
|
+
thread = Thread.start { yield } if block_given?
|
30
|
+
thread.abort_on_exception if thread
|
31
|
+
worker.run
|
32
|
+
ensure
|
33
|
+
thread.join(0.1) if thread
|
34
|
+
end
|
35
|
+
|
36
|
+
# Runs the worker until it has no more jobs to process,
|
37
|
+
# effectively drainig its queues.
|
38
|
+
def drain_worker_queues(worker)
|
39
|
+
worker.extend Module.new {
|
40
|
+
# For the child: stop as soon as it can't pop more jobs.
|
41
|
+
def no_job_available
|
42
|
+
shutdown
|
43
|
+
end
|
44
|
+
|
45
|
+
# For the parent: when the child stops,
|
46
|
+
# don't try to restart it; shutdown instead.
|
47
|
+
def spawn_replacement_child(*)
|
48
|
+
shutdown
|
49
|
+
end
|
50
|
+
}
|
51
|
+
|
52
|
+
worker.run
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|