resque 1.23.0 → 1.23.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of resque might be problematic. Click here for more details.

@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7c520ddc0c19372cbe01b4b9a8748769aa251226
4
+ data.tar.gz: 2ef08bafba6cd858a8110ba93e1d9c5d4815655a
5
+ SHA512:
6
+ metadata.gz: 7c3c1c8f7651ffb5b7cd22587c8d1f2854b42ab0665b3ce3bbdf00b106d54459036a8f808379228ca772bb492ea5572ec2f9498ecca6bb44d8e59d2da3cbf736
7
+ data.tar.gz: c510d27db4fa8821b27b26a170dd87d7ccaf21f51b14df9e831e63b02cc194286eeeb55ff9447abb6cf964acbbf265e7bdfde651570c7fa86e46a5a2e7a4082a
data/HISTORY.md CHANGED
@@ -1,3 +1,13 @@
1
+ ## 1.24.0.pre (YYYY-MM-DD)
2
+
3
+ * Optional RedisMultiQueue failure backend, can be enabled with
4
+ FAILURE_BACKEND=redis_multi_queue env var (@tarcieri)
5
+ * resque:failures:sort rake task will migrate an existing "failed" queue into
6
+ separate failure queues per job queue, allowing an easy migration to
7
+ the RedisMultiQueue failure backend (@tarcieri)
8
+ * Disable forking completely with FORK_PER_JOB=false env var (@tarcieri)
9
+ * Report a failure when processes are killed with signals (@dylanahsmith)
10
+
1
11
  ## 1.23.0 (2012-10-01)
2
12
 
3
13
  * don't run `before_fork` hook if Resque can't fork (@kjwierenga, @tarcieri, #672, #697)
@@ -1,3 +1,4 @@
1
+ require 'logger'
1
2
  require 'redis/namespace'
2
3
 
3
4
  require 'resque/version'
@@ -9,6 +10,10 @@ require 'resque/failure/base'
9
10
 
10
11
  require 'resque/helpers'
11
12
  require 'resque/stat'
13
+ require 'resque/logging'
14
+ require 'resque/log_formatters/quiet_formatter'
15
+ require 'resque/log_formatters/verbose_formatter'
16
+ require 'resque/log_formatters/very_verbose_formatter'
12
17
  require 'resque/job'
13
18
  require 'resque/worker'
14
19
  require 'resque/plugin'
@@ -66,46 +71,72 @@ module Resque
66
71
  end
67
72
  end
68
73
 
74
+ # Set or retrieve the current logger object
75
+ attr_accessor :logger
76
+
69
77
  # The `before_first_fork` hook will be run in the **parent** process
70
78
  # only once, before forking to run the first job. Be careful- any
71
79
  # changes you make will be permanent for the lifespan of the
72
80
  # worker.
73
81
  #
74
- # Call with a block to set the hook.
75
- # Call with no arguments to return the hook.
82
+ # Call with a block to register a hook.
83
+ # Call with no arguments to return all registered hooks.
76
84
  def before_first_fork(&block)
77
- block ? (@before_first_fork = block) : @before_first_fork
85
+ block ? register_hook(:before_first_fork, block) : hooks(:before_first_fork)
78
86
  end
79
87
 
80
- # Set a proc that will be called in the parent process before the
81
- # worker forks for the first time.
82
- attr_writer :before_first_fork
88
+ # Register a before_first_fork proc.
89
+ def before_first_fork=(block)
90
+ register_hook(:before_first_fork, block)
91
+ end
83
92
 
84
93
  # The `before_fork` hook will be run in the **parent** process
85
94
  # before every job, so be careful- any changes you make will be
86
95
  # permanent for the lifespan of the worker.
87
96
  #
88
- # Call with a block to set the hook.
89
- # Call with no arguments to return the hook.
97
+ # Call with a block to register a hook.
98
+ # Call with no arguments to return all registered hooks.
90
99
  def before_fork(&block)
91
- block ? (@before_fork = block) : @before_fork
100
+ block ? register_hook(:before_fork, block) : hooks(:before_fork)
92
101
  end
93
102
 
94
- # Set the before_fork proc.
95
- attr_writer :before_fork
103
+ # Register a before_fork proc.
104
+ def before_fork=(block)
105
+ register_hook(:before_fork, block)
106
+ end
96
107
 
97
108
  # The `after_fork` hook will be run in the child process and is passed
98
109
  # the current job. Any changes you make, therefore, will only live as
99
110
  # long as the job currently being processed.
100
111
  #
101
- # Call with a block to set the hook.
102
- # Call with no arguments to return the hook.
112
+ # Call with a block to register a hook.
113
+ # Call with no arguments to return all registered hooks.
103
114
  def after_fork(&block)
104
- block ? (@after_fork = block) : @after_fork
115
+ block ? register_hook(:after_fork, block) : hooks(:after_fork)
116
+ end
117
+
118
+ # Register an after_fork proc.
119
+ def after_fork=(block)
120
+ register_hook(:after_fork, block)
121
+ end
122
+
123
+ # The `before_pause` hook will be run in the parent process before the
124
+ # worker has paused processing (via #pause_processing or SIGUSR2).
125
+ def before_pause(&block)
126
+ block ? register_hook(:before_pause, block) : hooks(:before_pause)
127
+ end
128
+
129
+ # Set the after_pause proc.
130
+ attr_writer :before_pause
131
+
132
+ # The `after_pause` hook will be run in the parent process after the
133
+ # worker has paused (via SIGCONT).
134
+ def after_pause(&block)
135
+ block ? register_hook(:after_pause, block) : hooks(:after_pause)
105
136
  end
106
137
 
107
- # Set the after_fork proc.
108
- attr_writer :after_fork
138
+ # Set the after_continue proc.
139
+ attr_writer :after_pause
109
140
 
110
141
  def to_s
111
142
  "Resque Client connected to #{redis_id}"
@@ -280,11 +311,13 @@ module Resque
280
311
  end
281
312
  return if before_hooks.any? { |result| result == false }
282
313
 
283
- Job.destroy(queue_from_class(klass), klass, *args)
314
+ destroyed = Job.destroy(queue_from_class(klass), klass, *args)
284
315
 
285
316
  Plugin.after_dequeue_hooks(klass).each do |hook|
286
317
  klass.send(hook, *args)
287
318
  end
319
+
320
+ destroyed
288
321
  end
289
322
 
290
323
  # Given a class, try to extrapolate an appropriate queue based on a
@@ -367,5 +400,32 @@ module Resque
367
400
  key.sub("#{redis.namespace}:", '')
368
401
  end
369
402
  end
403
+
404
+ private
405
+
406
+ # Register a new proc as a hook. If the block is nil this is the
407
+ # equivalent of removing all hooks of the given name.
408
+ #
409
+ # `name` is the hook that the block should be registered with.
410
+ def register_hook(name, block)
411
+ return clear_hooks(name) if block.nil?
412
+
413
+ @hooks ||= {}
414
+ @hooks[name] ||= []
415
+ @hooks[name] << block
416
+ end
417
+
418
+ # Clear all hooks given a hook name.
419
+ def clear_hooks(name)
420
+ @hooks && @hooks[name] = []
421
+ end
422
+
423
+ # Retrieve all hooks of a given name.
424
+ def hooks(name)
425
+ (@hooks && @hooks[name]) || []
426
+ end
370
427
  end
371
428
 
429
+ # Log to STDOUT by default
430
+ Resque.logger = Logger.new(STDOUT)
431
+ Resque.logger.formatter = Resque::QuietFormatter.new
@@ -32,21 +32,52 @@ module Resque
32
32
  # back to `Resque::Failure::Redis`
33
33
  def self.backend
34
34
  return @backend if @backend
35
- require 'resque/failure/redis'
36
- @backend = Failure::Redis
35
+
36
+ case ENV['FAILURE_BACKEND']
37
+ when 'redis_multi_queue'
38
+ require 'resque/failure/redis_multi_queue'
39
+ @backend = Failure::RedisMultiQueue
40
+ when 'redis', nil
41
+ require 'resque/failure/redis'
42
+ @backend = Failure::Redis
43
+ else
44
+ raise ArgumentError, "invalid failure backend: #{FAILURE_BACKEND}"
45
+ end
46
+ end
47
+
48
+ # Obtain the failure queue name for a given job queue
49
+ def self.failure_queue_name(job_queue_name)
50
+ name = "#{job_queue_name}_failed"
51
+ Resque.redis.sadd(:failed_queues, name)
52
+ name
53
+ end
54
+
55
+ # Obtain the job queue name for a given failure queue
56
+ def self.job_queue_name(failure_queue_name)
57
+ failure_queue_name.sub(/_failed$/, '')
58
+ end
59
+
60
+ # Returns an array of all the failed queues in the system
61
+ def self.queues
62
+ backend.queues
37
63
  end
38
64
 
39
65
  # Returns the int count of how many failures we have seen.
40
- def self.count
41
- backend.count
66
+ def self.count(queue = nil, class_name = nil)
67
+ backend.count(queue, class_name)
42
68
  end
43
69
 
44
70
  # Returns an array of all the failures, paginated.
45
71
  #
46
- # `start` is the int of the first item in the page, `count` is the
72
+ # `offset` is the int of the first item in the page, `limit` is the
47
73
  # number of items to return.
48
- def self.all(start = 0, count = 1)
49
- backend.all(start, count)
74
+ def self.all(offset = 0, limit = 1, queue = nil)
75
+ backend.all(offset, limit, queue)
76
+ end
77
+
78
+ # Iterate across all failures with the given options
79
+ def self.each(offset = 0, limit = self.count, queue = nil, class_name = nil, &block)
80
+ backend.each(offset, limit, queue, class_name, &block)
50
81
  end
51
82
 
52
83
  # The string url of the backend's web interface, if any.
@@ -55,42 +86,28 @@ module Resque
55
86
  end
56
87
 
57
88
  # Clear all failure jobs
58
- def self.clear
59
- backend.clear
89
+ def self.clear(queue = nil)
90
+ backend.clear(queue)
60
91
  end
61
92
 
62
- def self.requeue(index)
63
- backend.requeue(index)
93
+ def self.requeue(id)
94
+ backend.requeue(id)
64
95
  end
65
96
 
66
- def self.remove(index)
67
- backend.remove(index)
97
+ def self.remove(id)
98
+ backend.remove(id)
68
99
  end
69
100
 
70
101
  # Requeues all failed jobs in a specific queue.
71
102
  # Queue name should be a string.
72
103
  def self.requeue_queue(queue)
73
- i=0
74
- while job = Resque::Failure.all(i)
75
- if job['queue'] == queue
76
- Resque::Failure.requeue(i)
77
- end
78
- i+=1
79
- end
104
+ backend.requeue_queue(queue)
80
105
  end
81
106
 
82
107
  # Removes all failed jobs in a specific queue.
83
108
  # Queue name should be a string.
84
109
  def self.remove_queue(queue)
85
- i=0
86
- while job = Resque::Failure.all(i)
87
- if job['queue'] == queue
88
- # This will remove the failure from the array so do not increment the index.
89
- Resque::Failure.remove(i)
90
- else
91
- i+=1
92
- end
93
- end
110
+ backend.remove_queue(queue)
94
111
  end
95
112
  end
96
113
  end
@@ -32,15 +32,24 @@ module Resque
32
32
  end
33
33
 
34
34
  # The number of failures.
35
- def self.count
35
+ def self.count(queue = nil, class_name = nil)
36
36
  0
37
37
  end
38
38
 
39
+ # Returns an array of all available failure queues
40
+ def self.queues
41
+ []
42
+ end
43
+
39
44
  # Returns a paginated array of failure objects.
40
- def self.all(start = 0, count = 1)
45
+ def self.all(offset = 0, limit = 1)
41
46
  []
42
47
  end
43
48
 
49
+ # Iterate across failed objects
50
+ def self.each(offset = 0, limit = self.count)
51
+ end
52
+
44
53
  # A URL where someone can go to view failures.
45
54
  def self.url
46
55
  end
@@ -23,8 +23,8 @@ module Resque
23
23
  end
24
24
 
25
25
  # The number of failures.
26
- def self.count
27
- classes.first.count
26
+ def self.count(queue = nil, class_name = nil)
27
+ classes.first.count(queue, class_name)
28
28
  end
29
29
 
30
30
  # Returns a paginated array of failure objects.
@@ -17,29 +17,71 @@ module Resque
17
17
  Resque.redis.rpush(:failed, data)
18
18
  end
19
19
 
20
- def self.count
21
- Resque.redis.llen(:failed).to_i
20
+ def self.count(queue = nil, class_name = nil)
21
+ raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s != "failed"
22
+
23
+ if class_name
24
+ n = 0
25
+ each(0, count(queue), queue, class_name) { n += 1 }
26
+ n
27
+ else
28
+ Resque.redis.llen(:failed).to_i
29
+ end
30
+ end
31
+
32
+ def self.queues
33
+ [:failed]
34
+ end
35
+
36
+ def self.all(offset = 0, limit = 1, queue = nil)
37
+ raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s == "failed"
38
+ Resque.list_range(:failed, offset, limit)
22
39
  end
23
40
 
24
- def self.all(start = 0, count = 1)
25
- Resque.list_range(:failed, start, count)
41
+ def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil)
42
+ Array(all(offset, limit, queue)).each_with_index do |item, i|
43
+ if !class_name || (item['payload'] && item['payload']['class'] == class_name)
44
+ yield offset + i, item
45
+ end
46
+ end
26
47
  end
27
48
 
28
- def self.clear
49
+ def self.clear(queue = nil)
50
+ raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s == "failed"
29
51
  Resque.redis.del(:failed)
30
52
  end
31
53
 
32
- def self.requeue(index)
33
- item = all(index)
54
+ def self.requeue(id)
55
+ item = all(id)
34
56
  item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
35
- Resque.redis.lset(:failed, index, Resque.encode(item))
57
+ Resque.redis.lset(:failed, id, Resque.encode(item))
36
58
  Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
37
59
  end
38
60
 
39
- def self.remove(index)
40
- id = rand(0xffffff)
41
- Resque.redis.lset(:failed, index, id)
42
- Resque.redis.lrem(:failed, 1, id)
61
+ def self.remove(id)
62
+ sentinel = ""
63
+ Resque.redis.lset(:failed, id, sentinel)
64
+ Resque.redis.lrem(:failed, 1, sentinel)
65
+ end
66
+
67
+ def self.requeue_queue(queue)
68
+ i = 0
69
+ while job = all(i)
70
+ requeue(i) if job['queue'] == queue
71
+ i += 1
72
+ end
73
+ end
74
+
75
+ def self.remove_queue(queue)
76
+ i = 0
77
+ while job = all(i)
78
+ if job['queue'] == queue
79
+ # This will remove the failure from the array so do not increment the index.
80
+ remove(i)
81
+ else
82
+ i += 1
83
+ end
84
+ end
43
85
  end
44
86
 
45
87
  def filter_backtrace(backtrace)
@@ -0,0 +1,86 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that stores exceptions in Redis. Very simple but
4
+ # works out of the box, along with support in the Resque web app.
5
+ class RedisMultiQueue < Base
6
+ def save
7
+ data = {
8
+ :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
9
+ :payload => payload,
10
+ :exception => exception.class.to_s,
11
+ :error => UTF8Util.clean(exception.to_s),
12
+ :backtrace => filter_backtrace(Array(exception.backtrace)),
13
+ :worker => worker.to_s,
14
+ :queue => queue
15
+ }
16
+ data = Resque.encode(data)
17
+ Resque.redis.rpush(Resque::Failure.failure_queue_name(queue), data)
18
+ end
19
+
20
+ def self.count(queue = nil, class_name = nil)
21
+ if queue
22
+ if class_name
23
+ n = 0
24
+ each(0, count(queue), queue, class_name) { n += 1 }
25
+ n
26
+ else
27
+ Resque.redis.llen(queue).to_i
28
+ end
29
+ else
30
+ total = 0
31
+ queues.each { |q| total += count(q) }
32
+ total
33
+ end
34
+ end
35
+
36
+ def self.all(offset = 0, limit = 1, queue = :failed)
37
+ Resque.list_range(queue, offset, limit)
38
+ end
39
+
40
+ def self.queues
41
+ Array(Resque.redis.smembers(:failed_queues))
42
+ end
43
+
44
+ def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil)
45
+ items = all(offset, limit, queue)
46
+ items = [items] unless items.is_a? Array
47
+ items.each_with_index do |item, i|
48
+ if !class_name || (item['payload'] && item['payload']['class'] == class_name)
49
+ yield offset + i, item
50
+ end
51
+ end
52
+ end
53
+
54
+ def self.clear(queue = :failed)
55
+ Resque.redis.del(queue)
56
+ end
57
+
58
+ def self.requeue(id, queue = :failed)
59
+ item = all(id, 1, queue)
60
+ item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
61
+ Resque.redis.lset(queue, id, Resque.encode(item))
62
+ Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
63
+ end
64
+
65
+ def self.remove(id, queue = :failed)
66
+ sentinel = ""
67
+ Resque.redis.lset(queue, id, sentinel)
68
+ Resque.redis.lrem(queue, 1, sentinel)
69
+ end
70
+
71
+ def self.requeue_queue(queue)
72
+ failure_queue = Resque::Failure.failure_queue_name(queue)
73
+ each(0, count(failure_queue), failure_queue) { |id, _| requeue(id, failure_queue) }
74
+ end
75
+
76
+ def self.remove_queue(queue)
77
+ Resque.redis.del(Resque::Failure.failure_queue_name(queue))
78
+ end
79
+
80
+ def filter_backtrace(backtrace)
81
+ index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
82
+ backtrace.first(index.to_i)
83
+ end
84
+ end
85
+ end
86
+ end