resque_sqs 1.25.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/HISTORY.md +467 -0
- data/LICENSE +20 -0
- data/README.markdown +866 -0
- data/Rakefile +70 -0
- data/bin/resque-sqs +81 -0
- data/bin/resque-sqs-web +27 -0
- data/lib/resque_sqs/errors.rb +13 -0
- data/lib/resque_sqs/failure/airbrake.rb +33 -0
- data/lib/resque_sqs/failure/base.rb +73 -0
- data/lib/resque_sqs/failure/multiple.rb +59 -0
- data/lib/resque_sqs/failure/redis.rb +108 -0
- data/lib/resque_sqs/failure/redis_multi_queue.rb +89 -0
- data/lib/resque_sqs/failure.rb +113 -0
- data/lib/resque_sqs/helpers.rb +107 -0
- data/lib/resque_sqs/job.rb +346 -0
- data/lib/resque_sqs/log_formatters/quiet_formatter.rb +7 -0
- data/lib/resque_sqs/log_formatters/verbose_formatter.rb +7 -0
- data/lib/resque_sqs/log_formatters/very_verbose_formatter.rb +8 -0
- data/lib/resque_sqs/logging.rb +18 -0
- data/lib/resque_sqs/plugin.rb +66 -0
- data/lib/resque_sqs/server/helpers.rb +52 -0
- data/lib/resque_sqs/server/public/favicon.ico +0 -0
- data/lib/resque_sqs/server/public/idle.png +0 -0
- data/lib/resque_sqs/server/public/jquery-1.3.2.min.js +19 -0
- data/lib/resque_sqs/server/public/jquery.relatize_date.js +95 -0
- data/lib/resque_sqs/server/public/poll.png +0 -0
- data/lib/resque_sqs/server/public/ranger.js +78 -0
- data/lib/resque_sqs/server/public/reset.css +44 -0
- data/lib/resque_sqs/server/public/style.css +91 -0
- data/lib/resque_sqs/server/public/working.png +0 -0
- data/lib/resque_sqs/server/test_helper.rb +19 -0
- data/lib/resque_sqs/server/views/error.erb +1 -0
- data/lib/resque_sqs/server/views/failed.erb +29 -0
- data/lib/resque_sqs/server/views/failed_job.erb +50 -0
- data/lib/resque_sqs/server/views/failed_queues_overview.erb +24 -0
- data/lib/resque_sqs/server/views/key_sets.erb +19 -0
- data/lib/resque_sqs/server/views/key_string.erb +11 -0
- data/lib/resque_sqs/server/views/layout.erb +44 -0
- data/lib/resque_sqs/server/views/next_more.erb +22 -0
- data/lib/resque_sqs/server/views/overview.erb +4 -0
- data/lib/resque_sqs/server/views/queues.erb +58 -0
- data/lib/resque_sqs/server/views/stats.erb +62 -0
- data/lib/resque_sqs/server/views/workers.erb +109 -0
- data/lib/resque_sqs/server/views/working.erb +72 -0
- data/lib/resque_sqs/server.rb +271 -0
- data/lib/resque_sqs/stat.rb +57 -0
- data/lib/resque_sqs/tasks.rb +83 -0
- data/lib/resque_sqs/vendor/utf8_util/utf8_util_18.rb +91 -0
- data/lib/resque_sqs/vendor/utf8_util/utf8_util_19.rb +5 -0
- data/lib/resque_sqs/vendor/utf8_util.rb +20 -0
- data/lib/resque_sqs/version.rb +3 -0
- data/lib/resque_sqs/worker.rb +779 -0
- data/lib/resque_sqs.rb +479 -0
- data/lib/tasks/redis_sqs.rake +161 -0
- data/lib/tasks/resque_sqs.rake +2 -0
- data/test/airbrake_test.rb +27 -0
- data/test/failure_base_test.rb +15 -0
- data/test/job_hooks_test.rb +465 -0
- data/test/job_plugins_test.rb +230 -0
- data/test/logging_test.rb +24 -0
- data/test/plugin_test.rb +116 -0
- data/test/redis-test-cluster.conf +115 -0
- data/test/redis-test.conf +115 -0
- data/test/resque-web_test.rb +59 -0
- data/test/resque_failure_redis_test.rb +19 -0
- data/test/resque_hook_test.rb +165 -0
- data/test/resque_test.rb +278 -0
- data/test/stdout +42 -0
- data/test/test_helper.rb +228 -0
- data/test/worker_test.rb +1080 -0
- metadata +202 -0
@@ -0,0 +1,779 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'set'
|
3
|
+
|
4
|
+
module ResqueSqs
|
5
|
+
# A Resque Worker processes jobs. On platforms that support fork(2),
|
6
|
+
# the worker will fork off a child to process each job. This ensures
|
7
|
+
# a clean slate when beginning the next job and cuts down on gradual
|
8
|
+
# memory growth as well as low level failures.
|
9
|
+
#
|
10
|
+
# It also ensures workers are always listening to signals from you,
|
11
|
+
# their master, and can react accordingly.
|
12
|
+
class Worker
|
13
|
+
include ResqueSqs::Logging
|
14
|
+
|
15
|
+
def redis
|
16
|
+
ResqueSqs.redis
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.redis
|
20
|
+
ResqueSqs.redis
|
21
|
+
end
|
22
|
+
|
23
|
+
# Given a Ruby object, returns a string suitable for storage in a
|
24
|
+
# queue.
|
25
|
+
def encode(object)
|
26
|
+
if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
|
27
|
+
MultiJson.dump object
|
28
|
+
else
|
29
|
+
MultiJson.encode object
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Given a string, returns a Ruby object.
|
34
|
+
def decode(object)
|
35
|
+
return unless object
|
36
|
+
|
37
|
+
begin
|
38
|
+
if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
|
39
|
+
MultiJson.load object
|
40
|
+
else
|
41
|
+
MultiJson.decode object
|
42
|
+
end
|
43
|
+
rescue ::MultiJson::DecodeError => e
|
44
|
+
raise DecodeException, e.message, e.backtrace
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Boolean indicating whether this worker can or can not fork.
|
49
|
+
# Automatically set if a fork(2) fails.
|
50
|
+
attr_accessor :cant_fork
|
51
|
+
|
52
|
+
attr_accessor :term_timeout
|
53
|
+
|
54
|
+
# decide whether to use new_kill_child logic
|
55
|
+
attr_accessor :term_child
|
56
|
+
|
57
|
+
# When set to true, forked workers will exit with `exit`, calling any `at_exit` code handlers that have been
|
58
|
+
# registered in the application. Otherwise, forked workers exit with `exit!`
|
59
|
+
attr_accessor :run_at_exit_hooks
|
60
|
+
|
61
|
+
attr_writer :to_s
|
62
|
+
|
63
|
+
# Returns an array of all worker objects.
|
64
|
+
def self.all
|
65
|
+
Array(redis.smembers(:workers)).map { |id| find(id) }.compact
|
66
|
+
end
|
67
|
+
|
68
|
+
# Returns an array of all worker objects currently processing
|
69
|
+
# jobs.
|
70
|
+
def self.working
|
71
|
+
names = all
|
72
|
+
return [] unless names.any?
|
73
|
+
|
74
|
+
names.map! { |name| "worker:#{name}" }
|
75
|
+
|
76
|
+
reportedly_working = {}
|
77
|
+
|
78
|
+
begin
|
79
|
+
reportedly_working = redis.mapped_mget(*names).reject do |key, value|
|
80
|
+
value.nil? || value.empty?
|
81
|
+
end
|
82
|
+
rescue Redis::Distributed::CannotDistribute
|
83
|
+
names.each do |name|
|
84
|
+
value = redis.get name
|
85
|
+
reportedly_working[name] = value unless value.nil? || value.empty?
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
reportedly_working.keys.map do |key|
|
90
|
+
find key.sub("worker:", '')
|
91
|
+
end.compact
|
92
|
+
end
|
93
|
+
|
94
|
+
# Returns a single worker object. Accepts a string id.
|
95
|
+
def self.find(worker_id)
|
96
|
+
if exists? worker_id
|
97
|
+
queues = worker_id.split(':')[-1].split(',')
|
98
|
+
worker = new(*queues)
|
99
|
+
worker.to_s = worker_id
|
100
|
+
worker
|
101
|
+
else
|
102
|
+
nil
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# Alias of `find`
|
107
|
+
def self.attach(worker_id)
|
108
|
+
find(worker_id)
|
109
|
+
end
|
110
|
+
|
111
|
+
# Given a string worker id, return a boolean indicating whether the
|
112
|
+
# worker exists
|
113
|
+
def self.exists?(worker_id)
|
114
|
+
redis.sismember(:workers, worker_id)
|
115
|
+
end
|
116
|
+
|
117
|
+
# Workers should be initialized with an array of string queue
|
118
|
+
# names. The order is important: a Worker will check the first
|
119
|
+
# queue given for a job. If none is found, it will check the
|
120
|
+
# second queue name given. If a job is found, it will be
|
121
|
+
# processed. Upon completion, the Worker will again check the
|
122
|
+
# first queue given, and so forth. In this way the queue list
|
123
|
+
# passed to a Worker on startup defines the priorities of queues.
|
124
|
+
#
|
125
|
+
# If passed a single "*", this Worker will operate on all queues
|
126
|
+
# in alphabetical order. Queues can be dynamically added or
|
127
|
+
# removed without needing to restart workers using this method.
|
128
|
+
def initialize(*queues)
|
129
|
+
@queues = queues.map { |queue| queue.to_s.strip }
|
130
|
+
@shutdown = nil
|
131
|
+
@paused = nil
|
132
|
+
validate_queues
|
133
|
+
end
|
134
|
+
|
135
|
+
# A worker must be given a queue, otherwise it won't know what to
|
136
|
+
# do with itself.
|
137
|
+
#
|
138
|
+
# You probably never need to call this.
|
139
|
+
def validate_queues
|
140
|
+
if @queues.nil? || @queues.empty?
|
141
|
+
raise NoQueueError.new("Please give each worker at least one queue.")
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
# This is the main workhorse method. Called on a Worker instance,
|
146
|
+
# it begins the worker life cycle.
|
147
|
+
#
|
148
|
+
# The following events occur during a worker's life cycle:
|
149
|
+
#
|
150
|
+
# 1. Startup: Signals are registered, dead workers are pruned,
|
151
|
+
# and this worker is registered.
|
152
|
+
# 2. Work loop: Jobs are pulled from a queue and processed.
|
153
|
+
# 3. Teardown: This worker is unregistered.
|
154
|
+
#
|
155
|
+
# Can be passed a float representing the polling frequency.
|
156
|
+
# The default is 5 seconds, but for a semi-active site you may
|
157
|
+
# want to use a smaller value.
|
158
|
+
#
|
159
|
+
# Also accepts a block which will be passed the job as soon as it
|
160
|
+
# has completed processing. Useful for testing.
|
161
|
+
def work(interval = 5.0, &block)
|
162
|
+
interval = Float(interval)
|
163
|
+
$0 = "resque_sqs: Starting"
|
164
|
+
startup
|
165
|
+
|
166
|
+
loop do
|
167
|
+
break if shutdown?
|
168
|
+
|
169
|
+
if not paused? and job = reserve
|
170
|
+
log "got: #{job.inspect}"
|
171
|
+
job.worker = self
|
172
|
+
working_on job
|
173
|
+
|
174
|
+
procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
|
175
|
+
if @child = fork(job)
|
176
|
+
srand # Reseeding
|
177
|
+
procline "Forked #{@child} at #{Time.now.to_i}"
|
178
|
+
begin
|
179
|
+
Process.waitpid(@child)
|
180
|
+
rescue SystemCallError
|
181
|
+
nil
|
182
|
+
end
|
183
|
+
job.fail(DirtyExit.new($?.to_s)) if $?.signaled?
|
184
|
+
else
|
185
|
+
unregister_signal_handlers if will_fork? && term_child
|
186
|
+
begin
|
187
|
+
|
188
|
+
reconnect
|
189
|
+
perform(job, &block)
|
190
|
+
|
191
|
+
rescue Exception => exception
|
192
|
+
report_failed_job(job,exception)
|
193
|
+
end
|
194
|
+
|
195
|
+
if will_fork?
|
196
|
+
run_at_exit_hooks ? exit : exit!
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
done_working
|
201
|
+
@child = nil
|
202
|
+
else
|
203
|
+
break if interval.zero?
|
204
|
+
log! "Sleeping for #{interval} seconds"
|
205
|
+
procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
|
206
|
+
sleep interval
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
unregister_worker
|
211
|
+
rescue Exception => exception
|
212
|
+
unless exception.class == SystemExit && !@child && run_at_exit_hooks
|
213
|
+
log "Failed to start worker : #{exception.inspect}"
|
214
|
+
|
215
|
+
unregister_worker(exception)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
# DEPRECATED. Processes a single job. If none is given, it will
|
220
|
+
# try to produce one. Usually run in the child.
|
221
|
+
def process(job = nil, &block)
|
222
|
+
return unless job ||= reserve
|
223
|
+
|
224
|
+
job.worker = self
|
225
|
+
working_on job
|
226
|
+
perform(job, &block)
|
227
|
+
ensure
|
228
|
+
done_working
|
229
|
+
end
|
230
|
+
|
231
|
+
# Reports the exception and marks the job as failed
|
232
|
+
def report_failed_job(job,exception)
|
233
|
+
log "#{job.inspect} failed: #{exception.inspect}"
|
234
|
+
begin
|
235
|
+
job.fail(exception)
|
236
|
+
rescue Object => exception
|
237
|
+
log "Received exception when reporting failure: #{exception.inspect}"
|
238
|
+
end
|
239
|
+
begin
|
240
|
+
failed!
|
241
|
+
rescue Object => exception
|
242
|
+
log "Received exception when increasing failed jobs counter (redis issue) : #{exception.inspect}"
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
# Processes a given job in the child.
|
247
|
+
def perform(job)
|
248
|
+
begin
|
249
|
+
run_hook :after_fork, job if will_fork?
|
250
|
+
job.perform
|
251
|
+
rescue Object => e
|
252
|
+
report_failed_job(job,e)
|
253
|
+
else
|
254
|
+
log "done: #{job.inspect}"
|
255
|
+
ensure
|
256
|
+
yield job if block_given?
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
# Attempts to grab a job off one of the provided queues. Returns
|
261
|
+
# nil if no job can be found.
|
262
|
+
def reserve
|
263
|
+
queues.each do |queue|
|
264
|
+
log! "Checking #{queue}"
|
265
|
+
if job = ResqueSqs.reserve(queue)
|
266
|
+
log! "Found job on #{queue}"
|
267
|
+
return job
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
nil
|
272
|
+
rescue Exception => e
|
273
|
+
log "Error reserving job: #{e.inspect}"
|
274
|
+
log e.backtrace.join("\n")
|
275
|
+
raise e
|
276
|
+
end
|
277
|
+
|
278
|
+
# Reconnect to Redis to avoid sharing a connection with the parent,
|
279
|
+
# retry up to 3 times with increasing delay before giving up.
|
280
|
+
def reconnect
|
281
|
+
tries = 0
|
282
|
+
begin
|
283
|
+
redis.client.reconnect
|
284
|
+
rescue Redis::BaseConnectionError
|
285
|
+
if (tries += 1) <= 3
|
286
|
+
log "Error reconnecting to Redis; retrying"
|
287
|
+
sleep(tries)
|
288
|
+
retry
|
289
|
+
else
|
290
|
+
log "Error reconnecting to Redis; quitting"
|
291
|
+
raise
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
# Returns a list of queues to use when searching for a job.
|
297
|
+
# A splat ("*") means you want every queue (in alpha order) - this
|
298
|
+
# can be useful for dynamically adding new queues.
|
299
|
+
def queues
|
300
|
+
@queues.map do |queue|
|
301
|
+
queue.strip!
|
302
|
+
if (matched_queues = glob_match(queue)).empty?
|
303
|
+
queue
|
304
|
+
else
|
305
|
+
matched_queues
|
306
|
+
end
|
307
|
+
end.flatten.uniq
|
308
|
+
end
|
309
|
+
|
310
|
+
def glob_match(pattern)
|
311
|
+
ResqueSqs.queues.select do |queue|
|
312
|
+
File.fnmatch?(pattern, queue)
|
313
|
+
end.sort
|
314
|
+
end
|
315
|
+
|
316
|
+
# Not every platform supports fork. Here we do our magic to
|
317
|
+
# determine if yours does.
|
318
|
+
def fork(job)
|
319
|
+
return if @cant_fork
|
320
|
+
|
321
|
+
# Only run before_fork hooks if we're actually going to fork
|
322
|
+
# (after checking @cant_fork)
|
323
|
+
run_hook :before_fork, job
|
324
|
+
|
325
|
+
begin
|
326
|
+
# IronRuby doesn't support `Kernel.fork` yet
|
327
|
+
if Kernel.respond_to?(:fork)
|
328
|
+
Kernel.fork if will_fork?
|
329
|
+
else
|
330
|
+
raise NotImplementedError
|
331
|
+
end
|
332
|
+
rescue NotImplementedError
|
333
|
+
@cant_fork = true
|
334
|
+
nil
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
# Runs all the methods needed when a worker begins its lifecycle.
|
339
|
+
def startup
|
340
|
+
Kernel.warn "WARNING: This way of doing signal handling is now deprecated. Please see http://hone.heroku.com/resque/2012/08/21/resque-signals.html for more info." unless term_child or $TESTING
|
341
|
+
enable_gc_optimizations
|
342
|
+
register_signal_handlers
|
343
|
+
prune_dead_workers
|
344
|
+
run_hook :before_first_fork
|
345
|
+
register_worker
|
346
|
+
|
347
|
+
# Fix buffering so we can `rake resque_sqs:work > resque.log` and
|
348
|
+
# get output from the child in there.
|
349
|
+
$stdout.sync = true
|
350
|
+
end
|
351
|
+
|
352
|
+
# Enables GC Optimizations if you're running REE.
|
353
|
+
# http://www.rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
|
354
|
+
def enable_gc_optimizations
|
355
|
+
if GC.respond_to?(:copy_on_write_friendly=)
|
356
|
+
GC.copy_on_write_friendly = true
|
357
|
+
end
|
358
|
+
end
|
359
|
+
|
360
|
+
# Registers the various signal handlers a worker responds to.
|
361
|
+
#
|
362
|
+
# TERM: Shutdown immediately, stop processing jobs.
|
363
|
+
# INT: Shutdown immediately, stop processing jobs.
|
364
|
+
# QUIT: Shutdown after the current job has finished processing.
|
365
|
+
# USR1: Kill the forked child immediately, continue processing jobs.
|
366
|
+
# USR2: Don't process any new jobs
|
367
|
+
# CONT: Start processing jobs again after a USR2
|
368
|
+
def register_signal_handlers
|
369
|
+
trap('TERM') { shutdown! }
|
370
|
+
trap('INT') { shutdown! }
|
371
|
+
|
372
|
+
begin
|
373
|
+
trap('QUIT') { shutdown }
|
374
|
+
if term_child
|
375
|
+
trap('USR1') { new_kill_child }
|
376
|
+
else
|
377
|
+
trap('USR1') { kill_child }
|
378
|
+
end
|
379
|
+
trap('USR2') { pause_processing }
|
380
|
+
trap('CONT') { unpause_processing }
|
381
|
+
rescue ArgumentError
|
382
|
+
warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
|
383
|
+
end
|
384
|
+
|
385
|
+
log! "Registered signals"
|
386
|
+
end
|
387
|
+
|
388
|
+
def unregister_signal_handlers
|
389
|
+
trap('TERM') do
|
390
|
+
trap ('TERM') do
|
391
|
+
# ignore subsequent terms
|
392
|
+
end
|
393
|
+
raise TermException.new("SIGTERM")
|
394
|
+
end
|
395
|
+
trap('INT', 'DEFAULT')
|
396
|
+
|
397
|
+
begin
|
398
|
+
trap('QUIT', 'DEFAULT')
|
399
|
+
trap('USR1', 'DEFAULT')
|
400
|
+
trap('USR2', 'DEFAULT')
|
401
|
+
rescue ArgumentError
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
# Schedule this worker for shutdown. Will finish processing the
|
406
|
+
# current job.
|
407
|
+
def shutdown
|
408
|
+
log 'Exiting...'
|
409
|
+
@shutdown = true
|
410
|
+
end
|
411
|
+
|
412
|
+
# Kill the child and shutdown immediately.
|
413
|
+
# If not forking, abort this process.
|
414
|
+
def shutdown!
|
415
|
+
shutdown
|
416
|
+
if term_child
|
417
|
+
if fork_per_job?
|
418
|
+
new_kill_child
|
419
|
+
else
|
420
|
+
# Raise TermException in the same process
|
421
|
+
trap('TERM') do
|
422
|
+
# ignore subsequent terms
|
423
|
+
end
|
424
|
+
raise TermException.new("SIGTERM")
|
425
|
+
end
|
426
|
+
else
|
427
|
+
kill_child
|
428
|
+
end
|
429
|
+
end
|
430
|
+
|
431
|
+
# Should this worker shutdown as soon as current job is finished?
|
432
|
+
def shutdown?
|
433
|
+
@shutdown
|
434
|
+
end
|
435
|
+
|
436
|
+
# Kills the forked child immediately, without remorse. The job it
|
437
|
+
# is processing will not be completed.
|
438
|
+
def kill_child
|
439
|
+
if @child
|
440
|
+
log! "Killing child at #{@child}"
|
441
|
+
if `ps -o pid,state -p #{@child}`
|
442
|
+
Process.kill("KILL", @child) rescue nil
|
443
|
+
else
|
444
|
+
log! "Child #{@child} not found, restarting."
|
445
|
+
shutdown
|
446
|
+
end
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
450
|
+
# Kills the forked child immediately with minimal remorse. The job it
|
451
|
+
# is processing will not be completed. Send the child a TERM signal,
|
452
|
+
# wait 5 seconds, and then a KILL signal if it has not quit
|
453
|
+
def new_kill_child
|
454
|
+
if @child
|
455
|
+
unless Process.waitpid(@child, Process::WNOHANG)
|
456
|
+
log! "Sending TERM signal to child #{@child}"
|
457
|
+
Process.kill("TERM", @child)
|
458
|
+
(term_timeout.to_f * 10).round.times do |i|
|
459
|
+
sleep(0.1)
|
460
|
+
return if Process.waitpid(@child, Process::WNOHANG)
|
461
|
+
end
|
462
|
+
log! "Sending KILL signal to child #{@child}"
|
463
|
+
Process.kill("KILL", @child)
|
464
|
+
else
|
465
|
+
log! "Child #{@child} already quit."
|
466
|
+
end
|
467
|
+
end
|
468
|
+
rescue SystemCallError
|
469
|
+
log! "Child #{@child} already quit and reaped."
|
470
|
+
end
|
471
|
+
|
472
|
+
# are we paused?
|
473
|
+
def paused?
|
474
|
+
@paused
|
475
|
+
end
|
476
|
+
|
477
|
+
# Stop processing jobs after the current one has completed (if we're
|
478
|
+
# currently running one).
|
479
|
+
def pause_processing
|
480
|
+
log "USR2 received; pausing job processing"
|
481
|
+
@paused = true
|
482
|
+
end
|
483
|
+
|
484
|
+
# Start processing jobs again after a pause
|
485
|
+
def unpause_processing
|
486
|
+
log "CONT received; resuming job processing"
|
487
|
+
@paused = false
|
488
|
+
end
|
489
|
+
|
490
|
+
# Looks for any workers which should be running on this server
|
491
|
+
# and, if they're not, removes them from Redis.
|
492
|
+
#
|
493
|
+
# This is a form of garbage collection. If a server is killed by a
|
494
|
+
# hard shutdown, power failure, or something else beyond our
|
495
|
+
# control, the Resque workers will not die gracefully and therefore
|
496
|
+
# will leave stale state information in Redis.
|
497
|
+
#
|
498
|
+
# By checking the current Redis state against the actual
|
499
|
+
# environment, we can determine if Redis is old and clean it up a bit.
|
500
|
+
def prune_dead_workers
|
501
|
+
all_workers = Worker.all
|
502
|
+
known_workers = worker_pids unless all_workers.empty?
|
503
|
+
all_workers.each do |worker|
|
504
|
+
host, pid, worker_queues_raw = worker.id.split(':')
|
505
|
+
worker_queues = worker_queues_raw.split(",")
|
506
|
+
unless @queues.include?("*") || (worker_queues.to_set == @queues.to_set)
|
507
|
+
# If the worker we are trying to prune does not belong to the queues
|
508
|
+
# we are listening to, we should not touch it.
|
509
|
+
# Attempt to prune a worker from different queues may easily result in
|
510
|
+
# an unknown class exception, since that worker could easily be even
|
511
|
+
# written in different language.
|
512
|
+
next
|
513
|
+
end
|
514
|
+
next unless host == hostname
|
515
|
+
next if known_workers.include?(pid)
|
516
|
+
log! "Pruning dead worker: #{worker}"
|
517
|
+
worker.unregister_worker
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
# Registers ourself as a worker. Useful when entering the worker
|
522
|
+
# lifecycle on startup.
|
523
|
+
def register_worker
|
524
|
+
redis.pipelined do
|
525
|
+
redis.sadd(:workers, self)
|
526
|
+
started!
|
527
|
+
end
|
528
|
+
end
|
529
|
+
|
530
|
+
# Runs a named hook, passing along any arguments.
|
531
|
+
def run_hook(name, *args)
|
532
|
+
return unless hooks = ResqueSqs.send(name)
|
533
|
+
msg = "Running #{name} hooks"
|
534
|
+
msg << " with #{args.inspect}" if args.any?
|
535
|
+
log msg
|
536
|
+
|
537
|
+
hooks.each do |hook|
|
538
|
+
args.any? ? hook.call(*args) : hook.call
|
539
|
+
end
|
540
|
+
end
|
541
|
+
|
542
|
+
# Unregisters ourself as a worker. Useful when shutting down.
|
543
|
+
def unregister_worker(exception = nil)
|
544
|
+
# If we're still processing a job, make sure it gets logged as a
|
545
|
+
# failure.
|
546
|
+
if (hash = processing) && !hash.empty?
|
547
|
+
job = Job.new(hash['queue'], hash['payload'])
|
548
|
+
# Ensure the proper worker is attached to this job, even if
|
549
|
+
# it's not the precise instance that died.
|
550
|
+
job.worker = self
|
551
|
+
job.fail(exception || DirtyExit.new)
|
552
|
+
end
|
553
|
+
|
554
|
+
redis.pipelined do
|
555
|
+
redis.srem(:workers, self)
|
556
|
+
redis.del("worker:#{self}")
|
557
|
+
redis.del("worker:#{self}:started")
|
558
|
+
|
559
|
+
Stat.clear("processed:#{self}")
|
560
|
+
Stat.clear("failed:#{self}")
|
561
|
+
end
|
562
|
+
end
|
563
|
+
|
564
|
+
# Given a job, tells Redis we're working on it. Useful for seeing
|
565
|
+
# what workers are doing and when.
|
566
|
+
def working_on(job)
|
567
|
+
data = encode \
|
568
|
+
:queue => job.queue,
|
569
|
+
:run_at => Time.now.utc.iso8601,
|
570
|
+
:payload => job.payload
|
571
|
+
redis.set("worker:#{self}", data)
|
572
|
+
end
|
573
|
+
|
574
|
+
# Called when we are done working - clears our `working_on` state
|
575
|
+
# and tells Redis we processed a job.
|
576
|
+
def done_working
|
577
|
+
redis.pipelined do
|
578
|
+
processed!
|
579
|
+
redis.del("worker:#{self}")
|
580
|
+
end
|
581
|
+
end
|
582
|
+
|
583
|
+
# How many jobs has this worker processed? Returns an int.
|
584
|
+
def processed
|
585
|
+
Stat["processed:#{self}"]
|
586
|
+
end
|
587
|
+
|
588
|
+
# Tell Redis we've processed a job.
|
589
|
+
def processed!
|
590
|
+
Stat << "processed"
|
591
|
+
Stat << "processed:#{self}"
|
592
|
+
end
|
593
|
+
|
594
|
+
# How many failed jobs has this worker seen? Returns an int.
|
595
|
+
def failed
|
596
|
+
Stat["failed:#{self}"]
|
597
|
+
end
|
598
|
+
|
599
|
+
# Tells Redis we've failed a job.
|
600
|
+
def failed!
|
601
|
+
Stat << "failed"
|
602
|
+
Stat << "failed:#{self}"
|
603
|
+
end
|
604
|
+
|
605
|
+
# What time did this worker start? Returns an instance of `Time`
|
606
|
+
def started
|
607
|
+
redis.get "worker:#{self}:started"
|
608
|
+
end
|
609
|
+
|
610
|
+
# Tell Redis we've started
|
611
|
+
def started!
|
612
|
+
redis.set("worker:#{self}:started", Time.now.to_s)
|
613
|
+
end
|
614
|
+
|
615
|
+
# Returns a hash explaining the Job we're currently processing, if any.
|
616
|
+
def job
|
617
|
+
decode(redis.get("worker:#{self}")) || {}
|
618
|
+
end
|
619
|
+
alias_method :processing, :job
|
620
|
+
|
621
|
+
# Boolean - true if working, false if not
|
622
|
+
def working?
|
623
|
+
state == :working
|
624
|
+
end
|
625
|
+
|
626
|
+
# Boolean - true if idle, false if not
|
627
|
+
def idle?
|
628
|
+
state == :idle
|
629
|
+
end
|
630
|
+
|
631
|
+
def will_fork?
|
632
|
+
!@cant_fork && !$TESTING && fork_per_job?
|
633
|
+
end
|
634
|
+
|
635
|
+
def fork_per_job?
|
636
|
+
ENV["FORK_PER_JOB"] != 'false'
|
637
|
+
end
|
638
|
+
|
639
|
+
# Returns a symbol representing the current worker state,
|
640
|
+
# which can be either :working or :idle
|
641
|
+
def state
|
642
|
+
redis.exists("worker:#{self}") ? :working : :idle
|
643
|
+
end
|
644
|
+
|
645
|
+
# Is this worker the same as another worker?
|
646
|
+
def ==(other)
|
647
|
+
to_s == other.to_s
|
648
|
+
end
|
649
|
+
|
650
|
+
def inspect
|
651
|
+
"#<Worker #{to_s}>"
|
652
|
+
end
|
653
|
+
|
654
|
+
# The string representation is the same as the id for this worker
|
655
|
+
# instance. Can be used with `Worker.find`.
|
656
|
+
def to_s
|
657
|
+
@to_s ||= "#{hostname}:#{pid}:#{@queues.join(',')}"
|
658
|
+
end
|
659
|
+
alias_method :id, :to_s
|
660
|
+
|
661
|
+
# chomp'd hostname of this machine
|
662
|
+
def hostname
|
663
|
+
Socket.gethostname
|
664
|
+
end
|
665
|
+
|
666
|
+
# Returns Integer PID of running worker
|
667
|
+
def pid
|
668
|
+
@pid ||= Process.pid
|
669
|
+
end
|
670
|
+
|
671
|
+
# Returns an Array of string pids of all the other workers on this
|
672
|
+
# machine. Useful when pruning dead workers on startup.
|
673
|
+
def worker_pids
|
674
|
+
if RUBY_PLATFORM =~ /solaris/
|
675
|
+
solaris_worker_pids
|
676
|
+
elsif RUBY_PLATFORM =~ /mingw32/
|
677
|
+
windows_worker_pids
|
678
|
+
else
|
679
|
+
linux_worker_pids
|
680
|
+
end
|
681
|
+
end
|
682
|
+
|
683
|
+
# Returns an Array of string pids of all the other workers on this
|
684
|
+
# machine. Useful when pruning dead workers on startup.
|
685
|
+
def windows_worker_pids
|
686
|
+
tasklist_output = `tasklist /FI "IMAGENAME eq ruby.exe" /FO list`.encode("UTF-8", Encoding.locale_charmap)
|
687
|
+
tasklist_output.split($/).select { |line| line =~ /^PID:/}.collect{ |line| line.gsub /PID:\s+/, '' }
|
688
|
+
end
|
689
|
+
|
690
|
+
# Find Resque worker pids on Linux and OS X.
|
691
|
+
#
|
692
|
+
def linux_worker_pids
|
693
|
+
`ps -A -o pid,command | grep "[r]esque" | grep -v "resque-web"`.split("\n").map do |line|
|
694
|
+
line.split(' ')[0]
|
695
|
+
end
|
696
|
+
end
|
697
|
+
|
698
|
+
# Find Resque worker pids on Solaris.
|
699
|
+
#
|
700
|
+
# Returns an Array of string pids of all the other workers on this
|
701
|
+
# machine. Useful when pruning dead workers on startup.
|
702
|
+
def solaris_worker_pids
|
703
|
+
`ps -A -o pid,comm | grep "[r]uby" | grep -v "resque-web"`.split("\n").map do |line|
|
704
|
+
real_pid = line.split(' ')[0]
|
705
|
+
pargs_command = `pargs -a #{real_pid} 2>/dev/null | grep [r]esque | grep -v "resque-web"`
|
706
|
+
if pargs_command.split(':')[1] == " resque-#{ResqueSqs::Version}"
|
707
|
+
real_pid
|
708
|
+
end
|
709
|
+
end.compact
|
710
|
+
end
|
711
|
+
|
712
|
+
# Given a string, sets the procline ($0) and logs.
|
713
|
+
# Procline is always in the format of:
|
714
|
+
# resque-VERSION: STRING
|
715
|
+
def procline(string)
|
716
|
+
$0 = "resque-#{ResqueSqs::Version}: #{string}"
|
717
|
+
log! $0
|
718
|
+
end
|
719
|
+
|
720
|
+
# Log a message to ResqueSqs.logger
|
721
|
+
# can't use alias_method since info/debug are private methods
|
722
|
+
def log(message)
|
723
|
+
info(message)
|
724
|
+
end
|
725
|
+
|
726
|
+
def log!(message)
|
727
|
+
debug(message)
|
728
|
+
end
|
729
|
+
|
730
|
+
# Deprecated legacy methods for controlling the logging threshhold
|
731
|
+
# Use ResqueSqs.logger.level now, e.g.:
|
732
|
+
#
|
733
|
+
# ResqueSqs.logger.level = Logger::DEBUG
|
734
|
+
#
|
735
|
+
def verbose
|
736
|
+
logger_severity_deprecation_warning
|
737
|
+
@verbose
|
738
|
+
end
|
739
|
+
|
740
|
+
def very_verbose
|
741
|
+
logger_severity_deprecation_warning
|
742
|
+
@very_verbose
|
743
|
+
end
|
744
|
+
|
745
|
+
def verbose=(value);
|
746
|
+
logger_severity_deprecation_warning
|
747
|
+
|
748
|
+
if value && !very_verbose
|
749
|
+
ResqueSqs.logger.formatter = VerboseFormatter.new
|
750
|
+
elsif !value
|
751
|
+
ResqueSqs.logger.formatter = QuietFormatter.new
|
752
|
+
end
|
753
|
+
|
754
|
+
@verbose = value
|
755
|
+
end
|
756
|
+
|
757
|
+
def very_verbose=(value)
|
758
|
+
logger_severity_deprecation_warning
|
759
|
+
if value
|
760
|
+
ResqueSqs.logger.formatter = VeryVerboseFormatter.new
|
761
|
+
elsif !value && verbose
|
762
|
+
ResqueSqs.logger.formatter = VerboseFormatter.new
|
763
|
+
else
|
764
|
+
ResqueSqs.logger.formatter = QuietFormatter.new
|
765
|
+
end
|
766
|
+
|
767
|
+
@very_verbose = value
|
768
|
+
end
|
769
|
+
|
770
|
+
def logger_severity_deprecation_warning
|
771
|
+
return if $TESTING
|
772
|
+
return if $warned_logger_severity_deprecation
|
773
|
+
Kernel.warn "*** DEPRECATION WARNING: ResqueSqs::Worker#verbose and #very_verbose are deprecated. Please set ResqueSqs.logger.level instead"
|
774
|
+
Kernel.warn "Called from: #{caller[0..5].join("\n\t")}"
|
775
|
+
$warned_logger_severity_deprecation = true
|
776
|
+
nil
|
777
|
+
end
|
778
|
+
end
|
779
|
+
end
|