resque 1.23.0 → 1.23.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of resque might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/HISTORY.md +10 -0
- data/lib/resque.rb +77 -17
- data/lib/resque/failure.rb +46 -29
- data/lib/resque/failure/base.rb +11 -2
- data/lib/resque/failure/multiple.rb +2 -2
- data/lib/resque/failure/redis.rb +54 -12
- data/lib/resque/failure/redis_multi_queue.rb +86 -0
- data/lib/resque/failure/thoughtbot.rb +1 -1
- data/lib/resque/log_formatters/quiet_formatter.rb +7 -0
- data/lib/resque/log_formatters/verbose_formatter.rb +7 -0
- data/lib/resque/log_formatters/very_verbose_formatter.rb +8 -0
- data/lib/resque/logging.rb +18 -0
- data/lib/resque/server.rb +20 -0
- data/lib/resque/server/helpers.rb +48 -0
- data/lib/resque/server/public/jquery.relatize_date.js +4 -4
- data/lib/resque/server/public/ranger.js +9 -4
- data/lib/resque/server/public/style.css +9 -4
- data/lib/resque/server/views/failed.erb +20 -59
- data/lib/resque/server/views/failed_job.erb +50 -0
- data/lib/resque/server/views/failed_queues_overview.erb +24 -0
- data/lib/resque/server/views/queues.erb +14 -5
- data/lib/resque/tasks.rb +21 -2
- data/lib/resque/version.rb +1 -1
- data/lib/resque/worker.rb +67 -24
- data/test/dump.rdb +0 -0
- data/test/logging_test.rb +24 -0
- data/test/redis-test.conf +1 -1
- data/test/resque_failure_redis_test.rb +1 -1
- data/test/resque_hook_test.rb +153 -0
- data/test/resque_test.rb +4 -4
- data/test/test_helper.rb +14 -14
- data/test/worker_test.rb +163 -9
- metadata +127 -132
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7c520ddc0c19372cbe01b4b9a8748769aa251226
|
4
|
+
data.tar.gz: 2ef08bafba6cd858a8110ba93e1d9c5d4815655a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7c3c1c8f7651ffb5b7cd22587c8d1f2854b42ab0665b3ce3bbdf00b106d54459036a8f808379228ca772bb492ea5572ec2f9498ecca6bb44d8e59d2da3cbf736
|
7
|
+
data.tar.gz: c510d27db4fa8821b27b26a170dd87d7ccaf21f51b14df9e831e63b02cc194286eeeb55ff9447abb6cf964acbbf265e7bdfde651570c7fa86e46a5a2e7a4082a
|
data/HISTORY.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
## 1.24.0.pre (YYYY-MM-DD)
|
2
|
+
|
3
|
+
* Optional RedisMultiQueue failure backend, can be enabled with
|
4
|
+
FAILURE_BACKEND=redis_multi_queue env var (@tarcieri)
|
5
|
+
* resque:failures:sort rake task will migrate an existing "failed" queue into
|
6
|
+
separate failure queues per job queue, allowing an easy migration to
|
7
|
+
the RedisMultiQueue failure backend (@tarcieri)
|
8
|
+
* Disable forking completely with FORK_PER_JOB=false env var (@tarcieri)
|
9
|
+
* Report a failure when processes are killed with signals (@dylanahsmith)
|
10
|
+
|
1
11
|
## 1.23.0 (2012-10-01)
|
2
12
|
|
3
13
|
* don't run `before_fork` hook if Resque can't fork (@kjwierenga, @tarcieri, #672, #697)
|
data/lib/resque.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'logger'
|
1
2
|
require 'redis/namespace'
|
2
3
|
|
3
4
|
require 'resque/version'
|
@@ -9,6 +10,10 @@ require 'resque/failure/base'
|
|
9
10
|
|
10
11
|
require 'resque/helpers'
|
11
12
|
require 'resque/stat'
|
13
|
+
require 'resque/logging'
|
14
|
+
require 'resque/log_formatters/quiet_formatter'
|
15
|
+
require 'resque/log_formatters/verbose_formatter'
|
16
|
+
require 'resque/log_formatters/very_verbose_formatter'
|
12
17
|
require 'resque/job'
|
13
18
|
require 'resque/worker'
|
14
19
|
require 'resque/plugin'
|
@@ -66,46 +71,72 @@ module Resque
|
|
66
71
|
end
|
67
72
|
end
|
68
73
|
|
74
|
+
# Set or retrieve the current logger object
|
75
|
+
attr_accessor :logger
|
76
|
+
|
69
77
|
# The `before_first_fork` hook will be run in the **parent** process
|
70
78
|
# only once, before forking to run the first job. Be careful- any
|
71
79
|
# changes you make will be permanent for the lifespan of the
|
72
80
|
# worker.
|
73
81
|
#
|
74
|
-
# Call with a block to
|
75
|
-
# Call with no arguments to return
|
82
|
+
# Call with a block to register a hook.
|
83
|
+
# Call with no arguments to return all registered hooks.
|
76
84
|
def before_first_fork(&block)
|
77
|
-
block ? (
|
85
|
+
block ? register_hook(:before_first_fork, block) : hooks(:before_first_fork)
|
78
86
|
end
|
79
87
|
|
80
|
-
#
|
81
|
-
|
82
|
-
|
88
|
+
# Register a before_first_fork proc.
|
89
|
+
def before_first_fork=(block)
|
90
|
+
register_hook(:before_first_fork, block)
|
91
|
+
end
|
83
92
|
|
84
93
|
# The `before_fork` hook will be run in the **parent** process
|
85
94
|
# before every job, so be careful- any changes you make will be
|
86
95
|
# permanent for the lifespan of the worker.
|
87
96
|
#
|
88
|
-
# Call with a block to
|
89
|
-
# Call with no arguments to return
|
97
|
+
# Call with a block to register a hook.
|
98
|
+
# Call with no arguments to return all registered hooks.
|
90
99
|
def before_fork(&block)
|
91
|
-
block ? (
|
100
|
+
block ? register_hook(:before_fork, block) : hooks(:before_fork)
|
92
101
|
end
|
93
102
|
|
94
|
-
#
|
95
|
-
|
103
|
+
# Register a before_fork proc.
|
104
|
+
def before_fork=(block)
|
105
|
+
register_hook(:before_fork, block)
|
106
|
+
end
|
96
107
|
|
97
108
|
# The `after_fork` hook will be run in the child process and is passed
|
98
109
|
# the current job. Any changes you make, therefore, will only live as
|
99
110
|
# long as the job currently being processed.
|
100
111
|
#
|
101
|
-
# Call with a block to
|
102
|
-
# Call with no arguments to return
|
112
|
+
# Call with a block to register a hook.
|
113
|
+
# Call with no arguments to return all registered hooks.
|
103
114
|
def after_fork(&block)
|
104
|
-
block ? (
|
115
|
+
block ? register_hook(:after_fork, block) : hooks(:after_fork)
|
116
|
+
end
|
117
|
+
|
118
|
+
# Register an after_fork proc.
|
119
|
+
def after_fork=(block)
|
120
|
+
register_hook(:after_fork, block)
|
121
|
+
end
|
122
|
+
|
123
|
+
# The `before_pause` hook will be run in the parent process before the
|
124
|
+
# worker has paused processing (via #pause_processing or SIGUSR2).
|
125
|
+
def before_pause(&block)
|
126
|
+
block ? register_hook(:before_pause, block) : hooks(:before_pause)
|
127
|
+
end
|
128
|
+
|
129
|
+
# Set the after_pause proc.
|
130
|
+
attr_writer :before_pause
|
131
|
+
|
132
|
+
# The `after_pause` hook will be run in the parent process after the
|
133
|
+
# worker has paused (via SIGCONT).
|
134
|
+
def after_pause(&block)
|
135
|
+
block ? register_hook(:after_pause, block) : hooks(:after_pause)
|
105
136
|
end
|
106
137
|
|
107
|
-
# Set the
|
108
|
-
attr_writer :
|
138
|
+
# Set the after_continue proc.
|
139
|
+
attr_writer :after_pause
|
109
140
|
|
110
141
|
def to_s
|
111
142
|
"Resque Client connected to #{redis_id}"
|
@@ -280,11 +311,13 @@ module Resque
|
|
280
311
|
end
|
281
312
|
return if before_hooks.any? { |result| result == false }
|
282
313
|
|
283
|
-
Job.destroy(queue_from_class(klass), klass, *args)
|
314
|
+
destroyed = Job.destroy(queue_from_class(klass), klass, *args)
|
284
315
|
|
285
316
|
Plugin.after_dequeue_hooks(klass).each do |hook|
|
286
317
|
klass.send(hook, *args)
|
287
318
|
end
|
319
|
+
|
320
|
+
destroyed
|
288
321
|
end
|
289
322
|
|
290
323
|
# Given a class, try to extrapolate an appropriate queue based on a
|
@@ -367,5 +400,32 @@ module Resque
|
|
367
400
|
key.sub("#{redis.namespace}:", '')
|
368
401
|
end
|
369
402
|
end
|
403
|
+
|
404
|
+
private
|
405
|
+
|
406
|
+
# Register a new proc as a hook. If the block is nil this is the
|
407
|
+
# equivalent of removing all hooks of the given name.
|
408
|
+
#
|
409
|
+
# `name` is the hook that the block should be registered with.
|
410
|
+
def register_hook(name, block)
|
411
|
+
return clear_hooks(name) if block.nil?
|
412
|
+
|
413
|
+
@hooks ||= {}
|
414
|
+
@hooks[name] ||= []
|
415
|
+
@hooks[name] << block
|
416
|
+
end
|
417
|
+
|
418
|
+
# Clear all hooks given a hook name.
|
419
|
+
def clear_hooks(name)
|
420
|
+
@hooks && @hooks[name] = []
|
421
|
+
end
|
422
|
+
|
423
|
+
# Retrieve all hooks of a given name.
|
424
|
+
def hooks(name)
|
425
|
+
(@hooks && @hooks[name]) || []
|
426
|
+
end
|
370
427
|
end
|
371
428
|
|
429
|
+
# Log to STDOUT by default
|
430
|
+
Resque.logger = Logger.new(STDOUT)
|
431
|
+
Resque.logger.formatter = Resque::QuietFormatter.new
|
data/lib/resque/failure.rb
CHANGED
@@ -32,21 +32,52 @@ module Resque
|
|
32
32
|
# back to `Resque::Failure::Redis`
|
33
33
|
def self.backend
|
34
34
|
return @backend if @backend
|
35
|
-
|
36
|
-
|
35
|
+
|
36
|
+
case ENV['FAILURE_BACKEND']
|
37
|
+
when 'redis_multi_queue'
|
38
|
+
require 'resque/failure/redis_multi_queue'
|
39
|
+
@backend = Failure::RedisMultiQueue
|
40
|
+
when 'redis', nil
|
41
|
+
require 'resque/failure/redis'
|
42
|
+
@backend = Failure::Redis
|
43
|
+
else
|
44
|
+
raise ArgumentError, "invalid failure backend: #{FAILURE_BACKEND}"
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Obtain the failure queue name for a given job queue
|
49
|
+
def self.failure_queue_name(job_queue_name)
|
50
|
+
name = "#{job_queue_name}_failed"
|
51
|
+
Resque.redis.sadd(:failed_queues, name)
|
52
|
+
name
|
53
|
+
end
|
54
|
+
|
55
|
+
# Obtain the job queue name for a given failure queue
|
56
|
+
def self.job_queue_name(failure_queue_name)
|
57
|
+
failure_queue_name.sub(/_failed$/, '')
|
58
|
+
end
|
59
|
+
|
60
|
+
# Returns an array of all the failed queues in the system
|
61
|
+
def self.queues
|
62
|
+
backend.queues
|
37
63
|
end
|
38
64
|
|
39
65
|
# Returns the int count of how many failures we have seen.
|
40
|
-
def self.count
|
41
|
-
backend.count
|
66
|
+
def self.count(queue = nil, class_name = nil)
|
67
|
+
backend.count(queue, class_name)
|
42
68
|
end
|
43
69
|
|
44
70
|
# Returns an array of all the failures, paginated.
|
45
71
|
#
|
46
|
-
# `
|
72
|
+
# `offset` is the int of the first item in the page, `limit` is the
|
47
73
|
# number of items to return.
|
48
|
-
def self.all(
|
49
|
-
backend.all(
|
74
|
+
def self.all(offset = 0, limit = 1, queue = nil)
|
75
|
+
backend.all(offset, limit, queue)
|
76
|
+
end
|
77
|
+
|
78
|
+
# Iterate across all failures with the given options
|
79
|
+
def self.each(offset = 0, limit = self.count, queue = nil, class_name = nil, &block)
|
80
|
+
backend.each(offset, limit, queue, class_name, &block)
|
50
81
|
end
|
51
82
|
|
52
83
|
# The string url of the backend's web interface, if any.
|
@@ -55,42 +86,28 @@ module Resque
|
|
55
86
|
end
|
56
87
|
|
57
88
|
# Clear all failure jobs
|
58
|
-
def self.clear
|
59
|
-
backend.clear
|
89
|
+
def self.clear(queue = nil)
|
90
|
+
backend.clear(queue)
|
60
91
|
end
|
61
92
|
|
62
|
-
def self.requeue(
|
63
|
-
backend.requeue(
|
93
|
+
def self.requeue(id)
|
94
|
+
backend.requeue(id)
|
64
95
|
end
|
65
96
|
|
66
|
-
def self.remove(
|
67
|
-
backend.remove(
|
97
|
+
def self.remove(id)
|
98
|
+
backend.remove(id)
|
68
99
|
end
|
69
100
|
|
70
101
|
# Requeues all failed jobs in a specific queue.
|
71
102
|
# Queue name should be a string.
|
72
103
|
def self.requeue_queue(queue)
|
73
|
-
|
74
|
-
while job = Resque::Failure.all(i)
|
75
|
-
if job['queue'] == queue
|
76
|
-
Resque::Failure.requeue(i)
|
77
|
-
end
|
78
|
-
i+=1
|
79
|
-
end
|
104
|
+
backend.requeue_queue(queue)
|
80
105
|
end
|
81
106
|
|
82
107
|
# Removes all failed jobs in a specific queue.
|
83
108
|
# Queue name should be a string.
|
84
109
|
def self.remove_queue(queue)
|
85
|
-
|
86
|
-
while job = Resque::Failure.all(i)
|
87
|
-
if job['queue'] == queue
|
88
|
-
# This will remove the failure from the array so do not increment the index.
|
89
|
-
Resque::Failure.remove(i)
|
90
|
-
else
|
91
|
-
i+=1
|
92
|
-
end
|
93
|
-
end
|
110
|
+
backend.remove_queue(queue)
|
94
111
|
end
|
95
112
|
end
|
96
113
|
end
|
data/lib/resque/failure/base.rb
CHANGED
@@ -32,15 +32,24 @@ module Resque
|
|
32
32
|
end
|
33
33
|
|
34
34
|
# The number of failures.
|
35
|
-
def self.count
|
35
|
+
def self.count(queue = nil, class_name = nil)
|
36
36
|
0
|
37
37
|
end
|
38
38
|
|
39
|
+
# Returns an array of all available failure queues
|
40
|
+
def self.queues
|
41
|
+
[]
|
42
|
+
end
|
43
|
+
|
39
44
|
# Returns a paginated array of failure objects.
|
40
|
-
def self.all(
|
45
|
+
def self.all(offset = 0, limit = 1)
|
41
46
|
[]
|
42
47
|
end
|
43
48
|
|
49
|
+
# Iterate across failed objects
|
50
|
+
def self.each(offset = 0, limit = self.count)
|
51
|
+
end
|
52
|
+
|
44
53
|
# A URL where someone can go to view failures.
|
45
54
|
def self.url
|
46
55
|
end
|
data/lib/resque/failure/redis.rb
CHANGED
@@ -17,29 +17,71 @@ module Resque
|
|
17
17
|
Resque.redis.rpush(:failed, data)
|
18
18
|
end
|
19
19
|
|
20
|
-
def self.count
|
21
|
-
|
20
|
+
def self.count(queue = nil, class_name = nil)
|
21
|
+
raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s != "failed"
|
22
|
+
|
23
|
+
if class_name
|
24
|
+
n = 0
|
25
|
+
each(0, count(queue), queue, class_name) { n += 1 }
|
26
|
+
n
|
27
|
+
else
|
28
|
+
Resque.redis.llen(:failed).to_i
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.queues
|
33
|
+
[:failed]
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.all(offset = 0, limit = 1, queue = nil)
|
37
|
+
raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s == "failed"
|
38
|
+
Resque.list_range(:failed, offset, limit)
|
22
39
|
end
|
23
40
|
|
24
|
-
def self.
|
25
|
-
|
41
|
+
def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil)
|
42
|
+
Array(all(offset, limit, queue)).each_with_index do |item, i|
|
43
|
+
if !class_name || (item['payload'] && item['payload']['class'] == class_name)
|
44
|
+
yield offset + i, item
|
45
|
+
end
|
46
|
+
end
|
26
47
|
end
|
27
48
|
|
28
|
-
def self.clear
|
49
|
+
def self.clear(queue = nil)
|
50
|
+
raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s == "failed"
|
29
51
|
Resque.redis.del(:failed)
|
30
52
|
end
|
31
53
|
|
32
|
-
def self.requeue(
|
33
|
-
item = all(
|
54
|
+
def self.requeue(id)
|
55
|
+
item = all(id)
|
34
56
|
item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
|
35
|
-
Resque.redis.lset(:failed,
|
57
|
+
Resque.redis.lset(:failed, id, Resque.encode(item))
|
36
58
|
Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
|
37
59
|
end
|
38
60
|
|
39
|
-
def self.remove(
|
40
|
-
|
41
|
-
Resque.redis.lset(:failed,
|
42
|
-
Resque.redis.lrem(:failed, 1,
|
61
|
+
def self.remove(id)
|
62
|
+
sentinel = ""
|
63
|
+
Resque.redis.lset(:failed, id, sentinel)
|
64
|
+
Resque.redis.lrem(:failed, 1, sentinel)
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.requeue_queue(queue)
|
68
|
+
i = 0
|
69
|
+
while job = all(i)
|
70
|
+
requeue(i) if job['queue'] == queue
|
71
|
+
i += 1
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.remove_queue(queue)
|
76
|
+
i = 0
|
77
|
+
while job = all(i)
|
78
|
+
if job['queue'] == queue
|
79
|
+
# This will remove the failure from the array so do not increment the index.
|
80
|
+
remove(i)
|
81
|
+
else
|
82
|
+
i += 1
|
83
|
+
end
|
84
|
+
end
|
43
85
|
end
|
44
86
|
|
45
87
|
def filter_backtrace(backtrace)
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# A Failure backend that stores exceptions in Redis. Very simple but
|
4
|
+
# works out of the box, along with support in the Resque web app.
|
5
|
+
class RedisMultiQueue < Base
|
6
|
+
def save
|
7
|
+
data = {
|
8
|
+
:failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
|
9
|
+
:payload => payload,
|
10
|
+
:exception => exception.class.to_s,
|
11
|
+
:error => UTF8Util.clean(exception.to_s),
|
12
|
+
:backtrace => filter_backtrace(Array(exception.backtrace)),
|
13
|
+
:worker => worker.to_s,
|
14
|
+
:queue => queue
|
15
|
+
}
|
16
|
+
data = Resque.encode(data)
|
17
|
+
Resque.redis.rpush(Resque::Failure.failure_queue_name(queue), data)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.count(queue = nil, class_name = nil)
|
21
|
+
if queue
|
22
|
+
if class_name
|
23
|
+
n = 0
|
24
|
+
each(0, count(queue), queue, class_name) { n += 1 }
|
25
|
+
n
|
26
|
+
else
|
27
|
+
Resque.redis.llen(queue).to_i
|
28
|
+
end
|
29
|
+
else
|
30
|
+
total = 0
|
31
|
+
queues.each { |q| total += count(q) }
|
32
|
+
total
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.all(offset = 0, limit = 1, queue = :failed)
|
37
|
+
Resque.list_range(queue, offset, limit)
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.queues
|
41
|
+
Array(Resque.redis.smembers(:failed_queues))
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil)
|
45
|
+
items = all(offset, limit, queue)
|
46
|
+
items = [items] unless items.is_a? Array
|
47
|
+
items.each_with_index do |item, i|
|
48
|
+
if !class_name || (item['payload'] && item['payload']['class'] == class_name)
|
49
|
+
yield offset + i, item
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.clear(queue = :failed)
|
55
|
+
Resque.redis.del(queue)
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.requeue(id, queue = :failed)
|
59
|
+
item = all(id, 1, queue)
|
60
|
+
item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
|
61
|
+
Resque.redis.lset(queue, id, Resque.encode(item))
|
62
|
+
Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.remove(id, queue = :failed)
|
66
|
+
sentinel = ""
|
67
|
+
Resque.redis.lset(queue, id, sentinel)
|
68
|
+
Resque.redis.lrem(queue, 1, sentinel)
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.requeue_queue(queue)
|
72
|
+
failure_queue = Resque::Failure.failure_queue_name(queue)
|
73
|
+
each(0, count(failure_queue), failure_queue) { |id, _| requeue(id, failure_queue) }
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.remove_queue(queue)
|
77
|
+
Resque.redis.del(Resque::Failure.failure_queue_name(queue))
|
78
|
+
end
|
79
|
+
|
80
|
+
def filter_backtrace(backtrace)
|
81
|
+
index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
|
82
|
+
backtrace.first(index.to_i)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|