reqless 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +8 -0
  3. data/README.md +648 -0
  4. data/Rakefile +117 -0
  5. data/bin/docker-build-and-test +22 -0
  6. data/exe/reqless-web +11 -0
  7. data/lib/reqless/config.rb +31 -0
  8. data/lib/reqless/failure_formatter.rb +43 -0
  9. data/lib/reqless/job.rb +496 -0
  10. data/lib/reqless/job_reservers/ordered.rb +29 -0
  11. data/lib/reqless/job_reservers/round_robin.rb +46 -0
  12. data/lib/reqless/job_reservers/shuffled_round_robin.rb +21 -0
  13. data/lib/reqless/lua/reqless-lib.lua +2965 -0
  14. data/lib/reqless/lua/reqless.lua +2545 -0
  15. data/lib/reqless/lua_script.rb +90 -0
  16. data/lib/reqless/middleware/requeue_exceptions.rb +94 -0
  17. data/lib/reqless/middleware/retry_exceptions.rb +72 -0
  18. data/lib/reqless/middleware/sentry.rb +66 -0
  19. data/lib/reqless/middleware/timeout.rb +63 -0
  20. data/lib/reqless/queue.rb +189 -0
  21. data/lib/reqless/queue_priority_pattern.rb +16 -0
  22. data/lib/reqless/server/static/css/bootstrap-responsive.css +686 -0
  23. data/lib/reqless/server/static/css/bootstrap-responsive.min.css +12 -0
  24. data/lib/reqless/server/static/css/bootstrap.css +3991 -0
  25. data/lib/reqless/server/static/css/bootstrap.min.css +689 -0
  26. data/lib/reqless/server/static/css/codemirror.css +112 -0
  27. data/lib/reqless/server/static/css/docs.css +839 -0
  28. data/lib/reqless/server/static/css/jquery.noty.css +105 -0
  29. data/lib/reqless/server/static/css/noty_theme_twitter.css +137 -0
  30. data/lib/reqless/server/static/css/style.css +200 -0
  31. data/lib/reqless/server/static/favicon.ico +0 -0
  32. data/lib/reqless/server/static/img/glyphicons-halflings-white.png +0 -0
  33. data/lib/reqless/server/static/img/glyphicons-halflings.png +0 -0
  34. data/lib/reqless/server/static/js/bootstrap-alert.js +94 -0
  35. data/lib/reqless/server/static/js/bootstrap-scrollspy.js +125 -0
  36. data/lib/reqless/server/static/js/bootstrap-tab.js +130 -0
  37. data/lib/reqless/server/static/js/bootstrap-tooltip.js +270 -0
  38. data/lib/reqless/server/static/js/bootstrap-typeahead.js +285 -0
  39. data/lib/reqless/server/static/js/bootstrap.js +1726 -0
  40. data/lib/reqless/server/static/js/bootstrap.min.js +6 -0
  41. data/lib/reqless/server/static/js/codemirror.js +2972 -0
  42. data/lib/reqless/server/static/js/jquery.noty.js +220 -0
  43. data/lib/reqless/server/static/js/mode/javascript.js +360 -0
  44. data/lib/reqless/server/static/js/theme/cobalt.css +18 -0
  45. data/lib/reqless/server/static/js/theme/eclipse.css +25 -0
  46. data/lib/reqless/server/static/js/theme/elegant.css +10 -0
  47. data/lib/reqless/server/static/js/theme/lesser-dark.css +45 -0
  48. data/lib/reqless/server/static/js/theme/monokai.css +28 -0
  49. data/lib/reqless/server/static/js/theme/neat.css +9 -0
  50. data/lib/reqless/server/static/js/theme/night.css +21 -0
  51. data/lib/reqless/server/static/js/theme/rubyblue.css +21 -0
  52. data/lib/reqless/server/static/js/theme/xq-dark.css +46 -0
  53. data/lib/reqless/server/views/_job.erb +259 -0
  54. data/lib/reqless/server/views/_job_list.erb +8 -0
  55. data/lib/reqless/server/views/_pagination.erb +7 -0
  56. data/lib/reqless/server/views/about.erb +130 -0
  57. data/lib/reqless/server/views/completed.erb +11 -0
  58. data/lib/reqless/server/views/config.erb +14 -0
  59. data/lib/reqless/server/views/failed.erb +48 -0
  60. data/lib/reqless/server/views/failed_type.erb +18 -0
  61. data/lib/reqless/server/views/job.erb +17 -0
  62. data/lib/reqless/server/views/layout.erb +451 -0
  63. data/lib/reqless/server/views/overview.erb +137 -0
  64. data/lib/reqless/server/views/queue.erb +125 -0
  65. data/lib/reqless/server/views/queues.erb +45 -0
  66. data/lib/reqless/server/views/tag.erb +6 -0
  67. data/lib/reqless/server/views/throttles.erb +38 -0
  68. data/lib/reqless/server/views/track.erb +75 -0
  69. data/lib/reqless/server/views/worker.erb +34 -0
  70. data/lib/reqless/server/views/workers.erb +14 -0
  71. data/lib/reqless/server.rb +549 -0
  72. data/lib/reqless/subscriber.rb +74 -0
  73. data/lib/reqless/test_helpers/worker_helpers.rb +55 -0
  74. data/lib/reqless/throttle.rb +57 -0
  75. data/lib/reqless/version.rb +5 -0
  76. data/lib/reqless/worker/base.rb +237 -0
  77. data/lib/reqless/worker/forking.rb +215 -0
  78. data/lib/reqless/worker/serial.rb +41 -0
  79. data/lib/reqless/worker.rb +5 -0
  80. data/lib/reqless.rb +309 -0
  81. metadata +399 -0
@@ -0,0 +1,549 @@
1
+ # Encoding: utf-8
2
+
3
+ require 'sinatra/base'
4
+ require 'reqless'
5
+
6
+ module Reqless
7
+ # The Reqless web interface
8
+ class Server < Sinatra::Base
9
+ # Path-y-ness
10
+ dir = File.dirname(File.expand_path(__FILE__))
11
+ set :views , "#{dir}/server/views"
12
+ set :public_folder, "#{dir}/server/static"
13
+
14
+ # For debugging purposes at least, I want this
15
+ set :reload_templates, true
16
+
17
+ # I'm not sure what this option is -- I'll look it up later
18
+ # set :static, true
19
+
20
+ attr_reader :client
21
+
22
+ def initialize(client)
23
+ @client = client
24
+ super
25
+ end
26
+
27
+ helpers do
28
+ include Rack::Utils
29
+
30
+ def url_path(*path_parts)
31
+ [path_prefix, path_parts].join('/').squeeze('/')
32
+ end
33
+ alias_method :u, :url_path
34
+
35
+ def path_prefix
36
+ request.env['SCRIPT_NAME']
37
+ end
38
+
39
+ def url_with_modified_query
40
+ url = URI(request.url)
41
+ existing_query = Rack::Utils.parse_query(url.query)
42
+ url.query = Rack::Utils.build_query(yield existing_query)
43
+ url.to_s
44
+ end
45
+
46
+ def page_url(offset)
47
+ url_with_modified_query do |query|
48
+ query.merge('page' => current_page + offset)
49
+ end
50
+ end
51
+
52
+ def next_page_url
53
+ page_url(1)
54
+ end
55
+
56
+ def prev_page_url
57
+ page_url(-1)
58
+ end
59
+
60
+ def current_page
61
+ @current_page ||= begin
62
+ Integer(params[:page])
63
+ rescue
64
+ 1
65
+ end
66
+ end
67
+
68
+ PAGE_SIZE = 25
69
+ def pagination_values
70
+ start = (current_page - 1) * PAGE_SIZE
71
+ [start, PAGE_SIZE]
72
+ end
73
+
74
+ def paginated(reqless_object, method, *args)
75
+ reqless_object.send(method, *(args + pagination_values))
76
+ end
77
+
78
+ def tabs
79
+ [
80
+ { name: 'Queues' , path: '/queues' },
81
+ { name: 'Throttles', path: '/throttles'},
82
+ { name: 'Workers' , path: '/workers' },
83
+ { name: 'Track' , path: '/track' },
84
+ { name: 'Failed' , path: '/failed' },
85
+ { name: 'Completed', path: '/completed'},
86
+ { name: 'Config' , path: '/config' },
87
+ { name: 'About' , path: '/about' }
88
+ ]
89
+ end
90
+
91
+ def application_name
92
+ client.config['application']
93
+ end
94
+
95
+ def queues
96
+ client.queues.counts
97
+ end
98
+
99
+ def throttles
100
+ client.throttles.counts
101
+ end
102
+
103
+ def tracked
104
+ client.jobs.tracked
105
+ end
106
+
107
+ def workers
108
+ client.workers.counts
109
+ end
110
+
111
+ def failed
112
+ client.jobs.failed
113
+ end
114
+
115
+ # Return the supplied object back as JSON
116
+ def json(obj)
117
+ content_type :json
118
+ obj.to_json
119
+ end
120
+
121
+ # Make the id acceptable as an id / att in HTML
122
+ def sanitize_attr(attr)
123
+ attr.gsub(/[^a-zA-Z\:\_]/, '-')
124
+ end
125
+
126
+ # What are the top tags? Since it might go on, say, every
127
+ # page, then we should probably be caching it
128
+ def top_tags
129
+ @top_tags ||= {
130
+ top: client.tags,
131
+ fetched: Time.now
132
+ }
133
+ if (Time.now - @top_tags[:fetched]) > 60
134
+ @top_tags = {
135
+ top: client.tags,
136
+ fetched: Time.now
137
+ }
138
+ end
139
+ @top_tags[:top]
140
+ end
141
+
142
+ def strftime(t)
143
+ # From http://stackoverflow.com/questions/195740
144
+ diff_seconds = Time.now - t
145
+ formatted = t.strftime('%b %e, %Y %H:%M:%S')
146
+ case diff_seconds
147
+ when 0 .. 59
148
+ "#{formatted} (#{diff_seconds.to_i} seconds ago)"
149
+ when 60 ... 3600
150
+ "#{formatted} (#{(diff_seconds / 60).to_i} minutes ago)"
151
+ when 3600 ... 3600 * 24
152
+ "#{formatted} (#{(diff_seconds / 3600).to_i} hours ago)"
153
+ when (3600 * 24) ... (3600 * 24 * 30)
154
+ "#{formatted} (#{(diff_seconds / (3600 * 24)).to_i} days ago)"
155
+ else
156
+ formatted
157
+ end
158
+ end
159
+ end
160
+
161
+ get '/?' do
162
+ erb :overview, layout: true, locals: { title: 'Overview' }
163
+ end
164
+
165
+ # Returns a JSON blob with the job counts for various queues
166
+ get '/queues.json' do
167
+ json(client.queues.counts)
168
+ end
169
+
170
+ get '/queues/?' do
171
+ erb :queues, layout: true, locals: {
172
+ title: 'Queues'
173
+ }
174
+ end
175
+
176
+ # Return the job counts for a specific queue
177
+ get '/queues/:name.json' do
178
+ json(client.queues[params[:name]].counts)
179
+ end
180
+
181
+ filtered_tabs = %w[ running throttled scheduled stalled depends recurring ].to_set
182
+ get '/queues/:name/?:tab?' do
183
+ queue = client.queues[params[:name]]
184
+ tab = params.fetch('tab', 'stats')
185
+
186
+ jobs = []
187
+ if tab == 'waiting'
188
+ jobs = queue.peek(*pagination_values)
189
+ elsif filtered_tabs.include?(tab)
190
+ jobs = paginated(queue.jobs, tab).map { |jid| client.jobs[jid] }
191
+ end
192
+
193
+ erb :queue, layout: true, locals: {
194
+ title: "Queue #{params[:name]}",
195
+ tab: tab,
196
+ jobs: jobs,
197
+ queue: client.queues[params[:name]].counts,
198
+ stats: queue.stats
199
+ }
200
+ end
201
+
202
+ get '/throttles/?' do
203
+ erb :throttles, layout: true, locals: {
204
+ title: 'Throttles'
205
+ }
206
+ end
207
+
208
+ post '/throttle' do
209
+ request.body.rewind
210
+ # Expects a JSON object: {'id': id, 'maximum': maximum}
211
+ data = JSON.parse(request.body.read)
212
+ if data['id'].nil? || data['maximum'].nil?
213
+ halt 400, 'Need throttle id and maximum value'
214
+ else
215
+ throttle = Throttle.new(data['id'], client)
216
+ throttle.maximum = data['maximum']
217
+ end
218
+ end
219
+
220
+ put '/throttle' do
221
+ request.body.rewind
222
+ # Expects a JSON object: {'id': id, 'expiration': expiration}
223
+ data = JSON.parse(request.body.read)
224
+ if data['id'].nil? || data['expiration'].nil?
225
+ halt 400, 'Need throttle id and expiration value'
226
+ else
227
+ throttle = Throttle.new(data['id'], client)
228
+ throttle.expiration = data['expiration']
229
+ end
230
+ end
231
+
232
+ delete '/throttle' do
233
+ request.body.rewind
234
+ # Expects a JSON object: {'id': id}
235
+ data = JSON.parse(request.body.read)
236
+ if data['id'].nil?
237
+ halt 400, 'Need throttle id'
238
+ else
239
+ throttle = Throttle.new(data['id'], client)
240
+ throttle.delete
241
+ return json({id: throttle.id, maximum: throttle.maximum})
242
+ end
243
+ end
244
+
245
+ get '/failed.json' do
246
+ json(client.jobs.failed)
247
+ end
248
+
249
+ get '/failed/?' do
250
+ # reqless-core doesn't provide functionality this way, so we'll
251
+ # do it ourselves. I'm not sure if this is how the core library
252
+ # should behave or not.
253
+ erb :failed, layout: true, locals: {
254
+ title: 'Failed',
255
+ failed: client.jobs.failed.keys.map do |t|
256
+ client.jobs.failed(t).tap { |f| f['type'] = t }
257
+ end
258
+ }
259
+ end
260
+
261
+ get '/failed/:type/?' do
262
+ erb :failed_type, layout: true, locals: {
263
+ title: 'Failed | ' + params[:type],
264
+ type: params[:type],
265
+ failed: paginated(client.jobs, :failed, params[:type])
266
+ }
267
+ end
268
+
269
+ get '/completed/?' do
270
+ completed = paginated(client.jobs, :complete)
271
+ erb :completed, layout: true, locals: {
272
+ title: 'Completed',
273
+ jobs: completed.map { |jid| client.jobs[jid] }
274
+ }
275
+ end
276
+
277
+ get '/track/?' do
278
+ erb :track, layout: true, locals: {
279
+ title: 'Track'
280
+ }
281
+ end
282
+
283
+ get '/jobs/:jid' do
284
+ erb :job, layout: true, locals: {
285
+ title: "Job | #{params[:jid]}",
286
+ jid: params[:jid],
287
+ job: client.jobs[params[:jid]]
288
+ }
289
+ end
290
+
291
+ get '/workers/?' do
292
+ erb :workers, layout: true, locals: {
293
+ title: 'Workers'
294
+ }
295
+ end
296
+
297
+ get '/workers/:worker' do
298
+ erb :worker, layout: true, locals: {
299
+ title: 'Worker | ' + params[:worker],
300
+ worker: client.workers[params[:worker]].tap do |w|
301
+ w['jobs'] = w['jobs'].map { |j| client.jobs[j] }
302
+ w['stalled'] = w['stalled'].map { |j| client.jobs[j] }
303
+ w['name'] = params[:worker]
304
+ end
305
+ }
306
+ end
307
+
308
+ get '/tag/?' do
309
+ jobs = paginated(client.jobs, :tagged, params[:tag])
310
+ erb :tag, layout: true, locals: {
311
+ title: "Tag | #{params[:tag]}",
312
+ tag: params[:tag],
313
+ jobs: jobs['jobs'].map { |jid| client.jobs[jid] },
314
+ total: jobs['total']
315
+ }
316
+ end
317
+
318
+ get '/config/?' do
319
+ erb :config, layout: true, locals: {
320
+ title: 'Config',
321
+ options: client.config.all
322
+ }
323
+ end
324
+
325
+ get '/about/?' do
326
+ erb :about, layout: true, locals: {
327
+ title: 'About'
328
+ }
329
+ end
330
+
331
+ # These are the bits where we accept AJAX requests
332
+ post '/track/?' do
333
+ request.body.rewind
334
+ # Expects a JSON-encoded hash with a job id, and optionally some tags
335
+ data = JSON.parse(request.body.read)
336
+ job = client.jobs[data['id']]
337
+ if !job.nil?
338
+ data.fetch('tags', false) ? job.track(*data['tags']) : job.track
339
+ if request.xhr?
340
+ json({ tracked: [job.jid] })
341
+ else
342
+ redirect to('/track')
343
+ end
344
+ else
345
+ if request.xhr?
346
+ json({ tracked: [] })
347
+ else
348
+ redirect to(request.referrer)
349
+ end
350
+ end
351
+ end
352
+
353
+ post '/untrack/?' do
354
+ request.body.rewind
355
+ # Expects a JSON-encoded array of job ids to stop tracking
356
+ jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
357
+ jobs.compact!
358
+ # Go ahead and cancel all the jobs!
359
+ jobs.each do |job|
360
+ job.untrack
361
+ end
362
+ return json({ untracked: jobs.map { |job| job.jid } })
363
+ end
364
+
365
+ post '/priority/?' do
366
+ request.body.rewind
367
+ # Expects a JSON-encoded dictionary of jid => priority
368
+ response = Hash.new
369
+ r = JSON.parse(request.body.read)
370
+ r.each_pair do |jid, priority|
371
+ begin
372
+ client.jobs[jid].priority = priority
373
+ response[jid] = priority
374
+ rescue
375
+ response[jid] = 'failed'
376
+ end
377
+ end
378
+ return json(response)
379
+ end
380
+
381
+ post '/pause/?' do
382
+ request.body.rewind
383
+ # Expects JSON blob: {'queue': <queue>}
384
+ r = JSON.parse(request.body.read)
385
+ if r['queue']
386
+ @client.queues[r['queue']].pause
387
+ return json({ queue: 'paused' })
388
+ else
389
+ raise 'No queue provided'
390
+ end
391
+ end
392
+
393
+ post '/unpause/?' do
394
+ request.body.rewind
395
+ # Expects JSON blob: {'queue': <queue>}
396
+ r = JSON.parse(request.body.read)
397
+ if r['queue']
398
+ @client.queues[r['queue']].unpause
399
+ return json({ queue: 'unpaused' })
400
+ else
401
+ raise 'No queue provided'
402
+ end
403
+ end
404
+
405
+ post '/timeout/?' do
406
+ request.body.rewind
407
+ # Expects JSON blob: {'jid': <jid>}
408
+ r = JSON.parse(request.body.read)
409
+ if r['jid']
410
+ @client.jobs[r['jid']].timeout
411
+ return json({ jid: r['jid'] })
412
+ else
413
+ raise 'No jid provided'
414
+ end
415
+ end
416
+
417
+ post '/tag/?' do
418
+ request.body.rewind
419
+ # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
420
+ response = Hash.new
421
+ JSON.parse(request.body.read).each_pair do |jid, tags|
422
+ begin
423
+ client.jobs[jid].tag(*tags)
424
+ response[jid] = tags
425
+ rescue
426
+ response[jid] = 'failed'
427
+ end
428
+ end
429
+ return json(response)
430
+ end
431
+
432
+ post '/untag/?' do
433
+ request.body.rewind
434
+ # Expects a JSON-encoded dictionary of jid => [tag, tag, tag]
435
+ response = Hash.new
436
+ JSON.parse(request.body.read).each_pair do |jid, tags|
437
+ begin
438
+ client.jobs[jid].untag(*tags)
439
+ response[jid] = tags
440
+ rescue
441
+ response[jid] = 'failed'
442
+ end
443
+ end
444
+ return json(response)
445
+ end
446
+
447
+ post '/move/?' do
448
+ request.body.rewind
449
+ # Expects a JSON-encoded hash of id: jid, and queue: queue_name
450
+ data = JSON.parse(request.body.read)
451
+ if data['id'].nil? || data['queue'].nil?
452
+ halt 400, 'Need id and queue arguments'
453
+ else
454
+ job = client.jobs[data['id']]
455
+ if job.nil?
456
+ halt 404, 'Could not find job'
457
+ else
458
+ job.requeue(data['queue'])
459
+ return json({ id: data['id'], queue: data['queue'] })
460
+ end
461
+ end
462
+ end
463
+
464
+ post '/undepend/?' do
465
+ request.body.rewind
466
+ # Expects a JSON-encoded hash of id: jid, and queue: queue_name
467
+ data = JSON.parse(request.body.read)
468
+ if data['id'].nil?
469
+ halt 400, 'Need id'
470
+ else
471
+ job = client.jobs[data['id']]
472
+ if job.nil?
473
+ halt 404, 'Could not find job'
474
+ else
475
+ job.undepend(data['dependency'])
476
+ return json({ id: data['id'] })
477
+ end
478
+ end
479
+ end
480
+
481
+ post '/retry/?' do
482
+ request.body.rewind
483
+ # Expects a JSON-encoded hash of id: jid, and queue: queue_name
484
+ data = JSON.parse(request.body.read)
485
+ if data['id'].nil?
486
+ halt 400, 'Need id'
487
+ else
488
+ job = client.jobs[data['id']]
489
+ if job.nil?
490
+ halt 404, 'Could not find job'
491
+ else
492
+ job.requeue(job.queue_name)
493
+ return json({ id: data['id'], queue: job.queue_name })
494
+ end
495
+ end
496
+ end
497
+
498
+ # Retry all the failures of a particular type
499
+ post '/retryall/?' do
500
+ request.body.rewind
501
+ # Expects a JSON-encoded hash of type: failure-type
502
+ data = JSON.parse(request.body.read)
503
+ if data['type'].nil?
504
+ halt 400, 'Neet type'
505
+ else
506
+ jobs = client.jobs.failed(data['type'], 0, 500)['jobs']
507
+ results = jobs.map do |job|
508
+ job.requeue(job.queue_name)
509
+ { id: job.jid, queue: job.queue_name }
510
+ end
511
+ return json(results)
512
+ end
513
+ end
514
+
515
+ post '/cancel/?' do
516
+ request.body.rewind
517
+ # Expects a JSON-encoded array of job ids to cancel
518
+ jobs = JSON.parse(request.body.read).map { |jid| client.jobs[jid] }
519
+ jobs.compact!
520
+ # Go ahead and cancel all the jobs!
521
+ jobs.each do |job|
522
+ job.cancel
523
+ end
524
+
525
+ if request.xhr?
526
+ return json({ canceled: jobs.map { |job| job.jid } })
527
+ else
528
+ redirect to(request.referrer)
529
+ end
530
+ end
531
+
532
+ post '/cancelall/?' do
533
+ request.body.rewind
534
+ # Expects a JSON-encoded hash of type: failure-type
535
+ data = JSON.parse(request.body.read)
536
+ if data['type'].nil?
537
+ halt 400, 'Neet type'
538
+ else
539
+ return json(client.jobs.failed(data['type'])['jobs'].map do |job|
540
+ job.cancel
541
+ { id: job.jid }
542
+ end)
543
+ end
544
+ end
545
+
546
+ # start the server if ruby file executed directly
547
+ run! if app_file == $PROGRAM_NAME
548
+ end
549
+ end
@@ -0,0 +1,74 @@
1
+ # Encoding: utf-8
2
+
3
+ require 'logger'
4
+ require 'thread'
5
+
6
+ module Reqless
7
+ # A class used for subscribing to messages in a thread
8
+ class Subscriber
9
+ def self.start(*args, &block)
10
+ new(*args, &block).tap(&:start)
11
+ end
12
+
13
+ attr_reader :channel, :redis
14
+
15
+ def initialize(client, channel, options = {}, &message_received_callback)
16
+ @channel = channel
17
+ @message_received_callback = message_received_callback
18
+ @log = options.fetch(:log) { ::Logger.new($stderr) }
19
+
20
+ # pub/sub blocks the connection so we must use a different redis
21
+ # connection
22
+ @client_redis = client.redis
23
+ @listener_redis = client.new_redis_connection
24
+
25
+ @my_channel = Reqless.generate_jid
26
+ end
27
+
28
+ # Start a thread listening
29
+ def start
30
+ queue = ::Queue.new
31
+
32
+ @thread = Thread.start do
33
+ begin
34
+ @listener_redis.subscribe(@channel, @my_channel) do |on|
35
+ on.subscribe do |channel|
36
+ # insert nil into the queue to indicate we've
37
+ # successfully subscribed
38
+ queue << nil if channel == @channel
39
+ end
40
+
41
+ on.message do |channel, message|
42
+ handle_message(channel, message)
43
+ end
44
+ end
45
+ # Watch for any exceptions so we don't block forever if
46
+ # subscribing to the channel fails
47
+ rescue Exception => e
48
+ queue << e
49
+ end
50
+ end
51
+
52
+ if (exception = queue.pop)
53
+ raise exception
54
+ end
55
+ end
56
+
57
+ def stop
58
+ @client_redis.publish(@my_channel, 'disconnect')
59
+ @thread.join
60
+ end
61
+
62
+ private
63
+
64
+ def handle_message(channel, message)
65
+ if channel == @my_channel
66
+ @listener_redis.unsubscribe(@channel, @my_channel) if message == "disconnect"
67
+ else
68
+ @message_received_callback.call(self, JSON.parse(message))
69
+ end
70
+ rescue Exception => error
71
+ @log.error("Reqless::Subscriber") { error }
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,55 @@
1
+ module Reqless
2
+ module WorkerHelpers
3
+ # Yield with a worker running, and then clean the worker up afterwards
4
+ def run_worker_concurrently_with(worker, &block)
5
+ thread = Thread.start { stop_worker_after(worker, &block) }
6
+ thread.abort_on_exception = true
7
+ worker.run
8
+ ensure
9
+ thread.join(0.1)
10
+ end
11
+
12
+ def stop_worker_after(worker, &block)
13
+ yield
14
+ ensure
15
+ worker.stop!
16
+ end
17
+
18
+ # Run only the given number of jobs, then stop
19
+ def run_jobs(worker, count)
20
+ worker.extend Module.new {
21
+ define_method(:jobs) do
22
+ base_enum = super()
23
+ Enumerator.new do |enum|
24
+ count.times { enum << base_enum.next }
25
+ end
26
+ end
27
+ }
28
+
29
+ thread = Thread.start { yield } if block_given?
30
+ thread.abort_on_exception if thread
31
+ worker.run
32
+ ensure
33
+ thread.join(0.1) if thread
34
+ end
35
+
36
+ # Runs the worker until it has no more jobs to process,
37
+ # effectively drainig its queues.
38
+ def drain_worker_queues(worker)
39
+ worker.extend Module.new {
40
+ # For the child: stop as soon as it can't pop more jobs.
41
+ def no_job_available
42
+ shutdown
43
+ end
44
+
45
+ # For the parent: when the child stops,
46
+ # don't try to restart it; shutdown instead.
47
+ def spawn_replacement_child(*)
48
+ shutdown
49
+ end
50
+ }
51
+
52
+ worker.run
53
+ end
54
+ end
55
+ end