resqueue 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/HISTORY.md +488 -0
  3. data/LICENSE +20 -0
  4. data/README.markdown +920 -0
  5. data/Rakefile +57 -0
  6. data/bin/resque +81 -0
  7. data/bin/resque-web +31 -0
  8. data/lib/resque.rb +578 -0
  9. data/lib/resque/data_store.rb +326 -0
  10. data/lib/resque/errors.rb +21 -0
  11. data/lib/resque/failure.rb +119 -0
  12. data/lib/resque/failure/airbrake.rb +33 -0
  13. data/lib/resque/failure/base.rb +73 -0
  14. data/lib/resque/failure/multiple.rb +68 -0
  15. data/lib/resque/failure/redis.rb +128 -0
  16. data/lib/resque/failure/redis_multi_queue.rb +104 -0
  17. data/lib/resque/helpers.rb +48 -0
  18. data/lib/resque/job.rb +296 -0
  19. data/lib/resque/log_formatters/quiet_formatter.rb +7 -0
  20. data/lib/resque/log_formatters/verbose_formatter.rb +7 -0
  21. data/lib/resque/log_formatters/very_verbose_formatter.rb +8 -0
  22. data/lib/resque/logging.rb +18 -0
  23. data/lib/resque/plugin.rb +78 -0
  24. data/lib/resque/server.rb +299 -0
  25. data/lib/resque/server/helpers.rb +64 -0
  26. data/lib/resque/server/public/favicon.ico +0 -0
  27. data/lib/resque/server/public/idle.png +0 -0
  28. data/lib/resque/server/public/jquery-1.12.4.min.js +5 -0
  29. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  30. data/lib/resque/server/public/poll.png +0 -0
  31. data/lib/resque/server/public/ranger.js +78 -0
  32. data/lib/resque/server/public/reset.css +44 -0
  33. data/lib/resque/server/public/style.css +91 -0
  34. data/lib/resque/server/public/working.png +0 -0
  35. data/lib/resque/server/test_helper.rb +19 -0
  36. data/lib/resque/server/views/error.erb +1 -0
  37. data/lib/resque/server/views/failed.erb +29 -0
  38. data/lib/resque/server/views/failed_job.erb +50 -0
  39. data/lib/resque/server/views/failed_queues_overview.erb +24 -0
  40. data/lib/resque/server/views/key_sets.erb +17 -0
  41. data/lib/resque/server/views/key_string.erb +11 -0
  42. data/lib/resque/server/views/layout.erb +44 -0
  43. data/lib/resque/server/views/next_more.erb +22 -0
  44. data/lib/resque/server/views/overview.erb +4 -0
  45. data/lib/resque/server/views/queues.erb +58 -0
  46. data/lib/resque/server/views/stats.erb +62 -0
  47. data/lib/resque/server/views/workers.erb +111 -0
  48. data/lib/resque/server/views/working.erb +72 -0
  49. data/lib/resque/stat.rb +58 -0
  50. data/lib/resque/tasks.rb +72 -0
  51. data/lib/resque/thread_signal.rb +45 -0
  52. data/lib/resque/vendor/utf8_util.rb +26 -0
  53. data/lib/resque/vendor/utf8_util/utf8_util_18.rb +91 -0
  54. data/lib/resque/vendor/utf8_util/utf8_util_19.rb +6 -0
  55. data/lib/resque/version.rb +3 -0
  56. data/lib/resque/worker.rb +892 -0
  57. data/lib/resqueue.rb +4 -0
  58. data/lib/tasks/redis.rake +161 -0
  59. data/lib/tasks/resque.rake +2 -0
  60. metadata +197 -0
@@ -0,0 +1,33 @@
1
+ begin
2
+ require 'airbrake'
3
+ rescue LoadError
4
+ raise "Can't find 'airbrake' gem. Please add it to your Gemfile or install it."
5
+ end
6
+
7
+ module Resque
8
+ module Failure
9
+ class Airbrake < Base
10
+ def self.configure(&block)
11
+ Resque.logger.warn "This actually sets global Airbrake configuration, " \
12
+ "which is probably not what you want. This will be gone in 2.0."
13
+ Resque::Failure.backend = self
14
+ ::Airbrake.configure(&block)
15
+ end
16
+
17
+ def self.count(queue = nil, class_name = nil)
18
+ # We can't get the total # of errors from Airbrake so we fake it
19
+ # by asking Resque how many errors it has seen.
20
+ Stat[:failed]
21
+ end
22
+
23
+ def save
24
+ ::Airbrake.notify(exception,
25
+ :parameters => {
26
+ :payload_class => payload['class'].to_s,
27
+ :payload_args => payload['args'].inspect
28
+ }
29
+ )
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,73 @@
1
+ module Resque
2
+ module Failure
3
+ # All Failure classes are expected to subclass Base.
4
+ #
5
+ # When a job fails, a new instance of your Failure backend is created
6
+ # and #save is called.
7
+ class Base
8
+ # The exception object raised by the failed job
9
+ attr_accessor :exception
10
+
11
+ # The worker object who detected the failure
12
+ attr_accessor :worker
13
+
14
+ # The string name of the queue from which the failed job was pulled
15
+ attr_accessor :queue
16
+
17
+ # The payload object associated with the failed job
18
+ attr_accessor :payload
19
+
20
+ def initialize(exception, worker, queue, payload)
21
+ @exception = exception
22
+ @worker = worker
23
+ @queue = queue
24
+ @payload = payload
25
+ end
26
+
27
+ # When a job fails, a new instance of your Failure backend is created
28
+ # and #save is called.
29
+ #
30
+ # This is where you POST or PUT or whatever to your Failure service.
31
+ def save
32
+ end
33
+
34
+ # The number of failures.
35
+ def self.count(queue = nil, class_name = nil)
36
+ 0
37
+ end
38
+
39
+ # Returns an array of all available failure queues
40
+ def self.queues
41
+ []
42
+ end
43
+
44
+ # Returns a paginated array of failure objects.
45
+ def self.all(offset = 0, limit = 1, queue = nil)
46
+ []
47
+ end
48
+
49
+ # Iterate across failed objects
50
+ def self.each(*args)
51
+ end
52
+
53
+ # A URL where someone can go to view failures.
54
+ def self.url
55
+ end
56
+
57
+ # Clear all failure objects
58
+ def self.clear(*args)
59
+ end
60
+
61
+ def self.requeue(*args)
62
+ end
63
+
64
+ def self.remove(*args)
65
+ end
66
+
67
+ # Logging!
68
+ def log(message)
69
+ @worker.log(message)
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,68 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that uses multiple backends
4
+ # delegates all queries to the first backend
5
+ class Multiple < Base
6
+
7
+ class << self
8
+ attr_accessor :classes
9
+ end
10
+
11
+ def self.configure
12
+ yield self
13
+ Resque::Failure.backend = self
14
+ end
15
+
16
+ def initialize(*args)
17
+ super
18
+ @backends = self.class.classes.map {|klass| klass.new(*args)}
19
+ end
20
+
21
+ def save
22
+ @backends.each(&:save)
23
+ end
24
+
25
+ # The number of failures.
26
+ def self.count(*args)
27
+ classes.first.count(*args)
28
+ end
29
+
30
+ # Returns an array of all available failure queues
31
+ def self.queues
32
+ classes.first.queues
33
+ end
34
+
35
+ # Returns a paginated array of failure objects.
36
+ def self.all(*args)
37
+ classes.first.all(*args)
38
+ end
39
+
40
+ # Iterate across failed objects
41
+ def self.each(*args, &block)
42
+ classes.first.each(*args, &block)
43
+ end
44
+
45
+ # A URL where someone can go to view failures.
46
+ def self.url
47
+ classes.first.url
48
+ end
49
+
50
+ # Clear all failure objects
51
+ def self.clear(*args)
52
+ classes.first.clear(*args)
53
+ end
54
+
55
+ def self.requeue(*args)
56
+ classes.first.requeue(*args)
57
+ end
58
+
59
+ def self.requeue_all
60
+ classes.first.requeue_all
61
+ end
62
+
63
+ def self.remove(index, queue)
64
+ classes.each { |klass| klass.remove(index) }
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,128 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that stores exceptions in Redis. Very simple but
4
+ # works out of the box, along with support in the Resque web app.
5
+ class Redis < Base
6
+
7
+ def data_store
8
+ Resque.data_store
9
+ end
10
+
11
+ def self.data_store
12
+ Resque.data_store
13
+ end
14
+
15
+ def save
16
+ data = {
17
+ :failed_at => UTF8Util.clean(Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")),
18
+ :payload => payload,
19
+ :exception => exception.class.to_s,
20
+ :error => UTF8Util.clean(exception.to_s),
21
+ :backtrace => filter_backtrace(Array(exception.backtrace)),
22
+ :worker => worker.to_s,
23
+ :queue => queue
24
+ }
25
+ data = Resque.encode(data)
26
+ data_store.push_to_failed_queue(data)
27
+ end
28
+
29
+ def self.count(queue = nil, class_name = nil)
30
+ check_queue(queue)
31
+
32
+ if class_name
33
+ n = 0
34
+ each(0, count(queue), queue, class_name) { n += 1 }
35
+ n
36
+ else
37
+ data_store.num_failed
38
+ end
39
+ end
40
+
41
+ def self.queues
42
+ data_store.failed_queue_names
43
+ end
44
+
45
+ def self.all(offset = 0, limit = 1, queue = nil)
46
+ check_queue(queue)
47
+ Resque.list_range(:failed, offset, limit)
48
+ end
49
+
50
+ def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
51
+ if class_name
52
+ original_limit = limit
53
+ limit = count
54
+ end
55
+ all_items = limit == 1 ? [all(offset,limit,queue)] : Array(all(offset, limit, queue))
56
+ reversed = false
57
+ if order.eql? 'desc'
58
+ all_items.reverse!
59
+ reversed = true
60
+ end
61
+ all_items.each_with_index do |item, i|
62
+ if !class_name || (item['payload'] && item['payload']['class'] == class_name && (original_limit -= 1) >= 0)
63
+ if reversed
64
+ id = (all_items.length - 1) + (offset - i)
65
+ else
66
+ id = offset + i
67
+ end
68
+ yield id, item
69
+ end
70
+ end
71
+ end
72
+
73
+ def self.clear(queue = nil)
74
+ check_queue(queue)
75
+ data_store.clear_failed_queue
76
+ end
77
+
78
+ def self.requeue(id, queue = nil)
79
+ check_queue(queue)
80
+ item = all(id)
81
+ item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
82
+ data_store.update_item_in_failed_queue(id,Resque.encode(item))
83
+ Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
84
+ end
85
+
86
+ def self.remove(id, queue = nil)
87
+ check_queue(queue)
88
+ data_store.remove_from_failed_queue(id, queue)
89
+ end
90
+
91
+ def self.requeue_queue(queue)
92
+ i = 0
93
+ while job = all(i)
94
+ requeue(i) if job['queue'] == queue
95
+ i += 1
96
+ end
97
+ end
98
+
99
+ def self.requeue_all
100
+ count.times do |num|
101
+ requeue(num)
102
+ end
103
+ end
104
+
105
+ def self.remove_queue(queue)
106
+ i = 0
107
+ while job = all(i)
108
+ if job['queue'] == queue
109
+ # This will remove the failure from the array so do not increment the index.
110
+ remove(i)
111
+ else
112
+ i += 1
113
+ end
114
+ end
115
+ end
116
+
117
+ def self.check_queue(queue)
118
+ raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s != "failed"
119
+ end
120
+
121
+ def filter_backtrace(backtrace)
122
+ index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
123
+ backtrace.first(index.to_i)
124
+ end
125
+
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,104 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that stores exceptions in Redis. Very simple but
4
+ # works out of the box, along with support in the Resque web app.
5
+ class RedisMultiQueue < Base
6
+
7
+ def data_store
8
+ Resque.data_store
9
+ end
10
+
11
+ def self.data_store
12
+ Resque.data_store
13
+ end
14
+
15
+ def save
16
+ data = {
17
+ :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
18
+ :payload => payload,
19
+ :exception => exception.class.to_s,
20
+ :error => UTF8Util.clean(exception.to_s),
21
+ :backtrace => filter_backtrace(Array(exception.backtrace)),
22
+ :worker => worker.to_s,
23
+ :queue => queue
24
+ }
25
+ data = Resque.encode(data)
26
+ data_store.push_to_failed_queue(data,Resque::Failure.failure_queue_name(queue))
27
+ end
28
+
29
+ def self.count(queue = nil, class_name = nil)
30
+ if queue
31
+ if class_name
32
+ n = 0
33
+ each(0, count(queue), queue, class_name) { n += 1 }
34
+ n
35
+ else
36
+ data_store.num_failed(queue).to_i
37
+ end
38
+ else
39
+ total = 0
40
+ queues.each { |q| total += count(q) }
41
+ total
42
+ end
43
+ end
44
+
45
+ def self.all(offset = 0, limit = 1, queue = :failed)
46
+ Resque.list_range(queue, offset, limit)
47
+ end
48
+
49
+ def self.queues
50
+ data_store.failed_queue_names(:failed_queues)
51
+ end
52
+
53
+ def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
54
+ items = all(offset, limit, queue)
55
+ items = [items] unless items.is_a? Array
56
+ reversed = false
57
+ if order.eql? 'desc'
58
+ items.reverse!
59
+ reversed = true
60
+ end
61
+ items.each_with_index do |item, i|
62
+ if !class_name || (item['payload'] && item['payload']['class'] == class_name)
63
+ id = reversed ? (items.length - 1) - (offset + i) : offset + i
64
+ yield id, item
65
+ end
66
+ end
67
+ end
68
+
69
+ def self.clear(queue = :failed)
70
+ queues = queue ? Array(queue) : self.queues
71
+ queues.each { |queue| data_store.clear_failed_queue(queue) }
72
+ end
73
+
74
+ def self.requeue(id, queue = :failed)
75
+ item = all(id, 1, queue)
76
+ item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
77
+ data_store.update_item_in_failed_queue(id,Resque.encode(item),queue)
78
+ Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
79
+ end
80
+
81
+ def self.remove(id, queue = :failed)
82
+ data_store.remove_from_queue(id,queue)
83
+ end
84
+
85
+ def self.requeue_queue(queue)
86
+ failure_queue = Resque::Failure.failure_queue_name(queue)
87
+ each(0, count(failure_queue), failure_queue) { |id, _| requeue(id, failure_queue) }
88
+ end
89
+
90
+ def self.requeue_all
91
+ queues.each { |queue| requeue_queue(Resque::Failure.job_queue_name(queue)) }
92
+ end
93
+
94
+ def self.remove_queue(queue)
95
+ data_store.remove_failed_queue(Resque::Failure.failure_queue_name(queue))
96
+ end
97
+
98
+ def filter_backtrace(backtrace)
99
+ index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
100
+ backtrace.first(index.to_i)
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,48 @@
1
+ require 'multi_json'
2
+
3
+ # OkJson won't work because it doesn't serialize symbols
4
+ # in the same way yajl and json do.
5
+ if MultiJson.respond_to?(:adapter)
6
+ raise "Please install the yajl-ruby or json gem" if MultiJson.adapter.to_s == 'MultiJson::Adapters::OkJson'
7
+ elsif MultiJson.respond_to?(:engine)
8
+ raise "Please install the yajl-ruby or json gem" if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
9
+ end
10
+
11
+ module Resque
12
+ # Methods used by various classes in Resque.
13
+ module Helpers
14
+ class DecodeException < StandardError; end
15
+
16
+ # Direct access to the Redis instance.
17
+ def redis
18
+ # No infinite recursions, please.
19
+ # Some external libraries depend on Resque::Helpers being mixed into
20
+ # Resque, but this method causes recursions. If we have a super method,
21
+ # assume it is canonical. (see #1150)
22
+ return super if defined?(super)
23
+
24
+ Resque.redis
25
+ end
26
+
27
+ # Given a Ruby object, returns a string suitable for storage in a
28
+ # queue.
29
+ def encode(object)
30
+ Resque.encode(object)
31
+ end
32
+
33
+ # Given a string, returns a Ruby object.
34
+ def decode(object)
35
+ Resque.decode(object)
36
+ end
37
+
38
+ # Given a word with dashes, returns a camel cased version of it.
39
+ def classify(dashed_word)
40
+ Resque.classify(dashed_word)
41
+ end
42
+
43
+ # Tries to find a constant with the name specified in the argument string
44
+ def constantize(camel_cased_word)
45
+ Resque.constantize(camel_cased_word)
46
+ end
47
+ end
48
+ end