resque-igo 1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. data/HISTORY.md +225 -0
  2. data/LICENSE +20 -0
  3. data/README.markdown +855 -0
  4. data/Rakefile +70 -0
  5. data/bin/resque +57 -0
  6. data/bin/resque-web +23 -0
  7. data/lib/resque.rb +380 -0
  8. data/lib/resque/errors.rb +10 -0
  9. data/lib/resque/failure.rb +66 -0
  10. data/lib/resque/failure/base.rb +61 -0
  11. data/lib/resque/failure/hoptoad.rb +132 -0
  12. data/lib/resque/failure/multiple.rb +50 -0
  13. data/lib/resque/failure/redis.rb +40 -0
  14. data/lib/resque/helpers.rb +71 -0
  15. data/lib/resque/job.rb +209 -0
  16. data/lib/resque/plugin.rb +51 -0
  17. data/lib/resque/server.rb +247 -0
  18. data/lib/resque/server/public/idle.png +0 -0
  19. data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
  20. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  21. data/lib/resque/server/public/poll.png +0 -0
  22. data/lib/resque/server/public/ranger.js +67 -0
  23. data/lib/resque/server/public/reset.css +48 -0
  24. data/lib/resque/server/public/style.css +86 -0
  25. data/lib/resque/server/public/working.png +0 -0
  26. data/lib/resque/server/test_helper.rb +19 -0
  27. data/lib/resque/server/views/error.erb +1 -0
  28. data/lib/resque/server/views/failed.erb +53 -0
  29. data/lib/resque/server/views/key_sets.erb +20 -0
  30. data/lib/resque/server/views/key_string.erb +11 -0
  31. data/lib/resque/server/views/layout.erb +42 -0
  32. data/lib/resque/server/views/next_more.erb +10 -0
  33. data/lib/resque/server/views/overview.erb +4 -0
  34. data/lib/resque/server/views/queues.erb +65 -0
  35. data/lib/resque/server/views/stats.erb +73 -0
  36. data/lib/resque/server/views/workers.erb +109 -0
  37. data/lib/resque/server/views/working.erb +68 -0
  38. data/lib/resque/stat.rb +54 -0
  39. data/lib/resque/tasks.rb +39 -0
  40. data/lib/resque/version.rb +3 -0
  41. data/lib/resque/worker.rb +478 -0
  42. data/tasks/resque.rake +2 -0
  43. data/test/job_hooks_test.rb +323 -0
  44. data/test/job_plugins_test.rb +230 -0
  45. data/test/plugin_test.rb +116 -0
  46. data/test/resque-web_test.rb +54 -0
  47. data/test/resque_test.rb +351 -0
  48. data/test/test_helper.rb +166 -0
  49. data/test/worker_test.rb +302 -0
  50. metadata +180 -0
@@ -0,0 +1,10 @@
1
+ module Resque
2
+ # Raised whenever we need a queue but none is provided.
3
+ class NoQueueError < RuntimeError; end
4
+
5
+ # Raised when trying to create a job without a class
6
+ class NoClassError < RuntimeError; end
7
+
8
+ # Raised when a worker was killed while processing a job.
9
+ class DirtyExit < RuntimeError; end
10
+ end
@@ -0,0 +1,66 @@
1
+ module Resque
2
+ # The Failure module provides an interface for working with different
3
+ # failure backends.
4
+ #
5
+ # You can use it to query the failure backend without knowing which specific
6
+ # backend is being used. For instance, the Resque web app uses it to display
7
+ # stats and other information.
8
+ module Failure
9
+ # Creates a new failure, which is delegated to the appropriate backend.
10
+ #
11
+ # Expects a hash with the following keys:
12
+ # :exception - The Exception object
13
+ # :worker - The Worker object who is reporting the failure
14
+ # :queue - The string name of the queue from which the job was pulled
15
+ # :payload - The job's payload
16
+ def self.create(options = {})
17
+ backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
18
+ end
19
+
20
+ #
21
+ # Sets the current backend. Expects a class descendent of
22
+ # `Resque::Failure::Base`.
23
+ #
24
+ # Example use:
25
+ # require 'resque/failure/hoptoad'
26
+ # Resque::Failure.backend = Resque::Failure::Hoptoad
27
+ def self.backend=(backend)
28
+ @backend = backend
29
+ end
30
+
31
+ # Returns the current backend class. If none has been set, falls
32
+ # back to `Resque::Failure::Redis`
33
+ def self.backend
34
+ return @backend if @backend
35
+ require 'resque/failure/redis'
36
+ @backend = Failure::Redis
37
+ end
38
+
39
+ # Returns the int count of how many failures we have seen.
40
+ def self.count
41
+ backend.count
42
+ end
43
+
44
+ # Returns an array of all the failures, paginated.
45
+ #
46
+ # `start` is the int of the first item in the page, `count` is the
47
+ # number of items to return.
48
+ def self.all(start = 0, count = 1)
49
+ backend.all(start, count)
50
+ end
51
+
52
+ # The string url of the backend's web interface, if any.
53
+ def self.url
54
+ backend.url
55
+ end
56
+
57
+ # Clear all failure jobs
58
+ def self.clear
59
+ backend.clear
60
+ end
61
+
62
+ def self.requeue(index)
63
+ backend.requeue(index)
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,61 @@
1
+ module Resque
2
+ module Failure
3
+ # All Failure classes are expected to subclass Base.
4
+ #
5
+ # When a job fails, a new instance of your Failure backend is created
6
+ # and #save is called.
7
+ class Base
8
+ # The exception object raised by the failed job
9
+ attr_accessor :exception
10
+
11
+ # The worker object who detected the failure
12
+ attr_accessor :worker
13
+
14
+ # The string name of the queue from which the failed job was pulled
15
+ attr_accessor :queue
16
+
17
+ # The payload object associated with the failed job
18
+ attr_accessor :payload
19
+
20
+ def initialize(exception, worker, queue, payload)
21
+ @exception = exception
22
+ @worker = worker
23
+ @queue = queue
24
+ @payload = payload
25
+ end
26
+
27
+ # When a job fails, a new instance of your Failure backend is created
28
+ # and #save is called.
29
+ #
30
+ # This is where you POST or PUT or whatever to your Failure service.
31
+ def save
32
+ end
33
+
34
+ # The number of failures.
35
+ def self.count
36
+ 0
37
+ end
38
+
39
+ # Returns a paginated array of failure objects.
40
+ def self.all(start = 0, count = 1)
41
+ []
42
+ end
43
+
44
+ # A URL where someone can go to view failures.
45
+ def self.url
46
+ end
47
+
48
+ # Clear all failure objects
49
+ def self.clear
50
+ end
51
+
52
+ def self.requeue(index)
53
+ end
54
+
55
+ # Logging!
56
+ def log(message)
57
+ @worker.log(message)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,132 @@
1
+ require 'net/https'
2
+ require 'builder'
3
+ require 'uri'
4
+
5
+ module Resque
6
+ module Failure
7
+ # A Failure backend that sends exceptions raised by jobs to Hoptoad.
8
+ #
9
+ # To use it, put this code in an initializer, Rake task, or wherever:
10
+ #
11
+ # require 'resque/failure/hoptoad'
12
+ #
13
+ # Resque::Failure::Hoptoad.configure do |config|
14
+ # config.api_key = 'blah'
15
+ # config.secure = true
16
+ #
17
+ # # optional proxy support
18
+ # config.proxy_host = 'x.y.z.t'
19
+ # config.proxy_port = 8080
20
+ #
21
+ # # server env support, defaults to RAILS_ENV or RACK_ENV
22
+ # config.server_environment = "test"
23
+ # end
24
+ class Hoptoad < Base
25
+ # From the hoptoad plugin
26
+ INPUT_FORMAT = /^([^:]+):(\d+)(?::in `([^']+)')?$/
27
+
28
+ class << self
29
+ attr_accessor :secure, :api_key, :proxy_host, :proxy_port
30
+ attr_accessor :server_environment
31
+ end
32
+
33
+ def self.count
34
+ # We can't get the total # of errors from Hoptoad so we fake it
35
+ # by asking Resque how many errors it has seen.
36
+ Stat[:failed]
37
+ end
38
+
39
+ def self.configure
40
+ yield self
41
+ Resque::Failure.backend = self
42
+ end
43
+
44
+ def save
45
+ http = use_ssl? ? :https : :http
46
+ url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
47
+
48
+ request = Net::HTTP::Proxy(self.class.proxy_host, self.class.proxy_port)
49
+ http = request.new(url.host, url.port)
50
+ headers = {
51
+ 'Content-type' => 'text/xml',
52
+ 'Accept' => 'text/xml, application/xml'
53
+ }
54
+
55
+ http.read_timeout = 5 # seconds
56
+ http.open_timeout = 2 # seconds
57
+
58
+ http.use_ssl = use_ssl?
59
+
60
+ begin
61
+ response = http.post(url.path, xml, headers)
62
+ rescue TimeoutError => e
63
+ log "Timeout while contacting the Hoptoad server."
64
+ end
65
+
66
+ case response
67
+ when Net::HTTPSuccess then
68
+ log "Hoptoad Success: #{response.class}"
69
+ else
70
+ body = response.body if response.respond_to? :body
71
+ log "Hoptoad Failure: #{response.class}\n#{body}"
72
+ end
73
+ end
74
+
75
+ def xml
76
+ x = Builder::XmlMarkup.new
77
+ x.instruct!
78
+ x.notice :version=>"2.0" do
79
+ x.tag! "api-key", api_key
80
+ x.notifier do
81
+ x.name "Resqueue"
82
+ x.version "0.1"
83
+ x.url "http://github.com/defunkt/resque"
84
+ end
85
+ x.error do
86
+ x.tag! "class", exception.class.name
87
+ x.message "#{exception.class.name}: #{exception.message}"
88
+ x.backtrace do
89
+ fill_in_backtrace_lines(x)
90
+ end
91
+ end
92
+ x.request do
93
+ x.url queue.to_s
94
+ x.component worker.to_s
95
+ x.params do
96
+ x.var :key=>"payload_class" do
97
+ x.text! payload["class"].to_s
98
+ end
99
+ x.var :key=>"payload_args" do
100
+ x.text! payload["args"].to_s
101
+ end
102
+ end
103
+ end
104
+ x.tag!("server-environment") do
105
+ x.tag!("environment-name",server_environment)
106
+ end
107
+
108
+ end
109
+ end
110
+
111
+ def fill_in_backtrace_lines(x)
112
+ Array(exception.backtrace).each do |unparsed_line|
113
+ _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
114
+ x.line :file => file,:number => number
115
+ end
116
+ end
117
+
118
+ def use_ssl?
119
+ self.class.secure
120
+ end
121
+
122
+ def api_key
123
+ self.class.api_key
124
+ end
125
+
126
+ def server_environment
127
+ return self.class.server_environment if self.class.server_environment
128
+ defined?(RAILS_ENV) ? RAILS_ENV : (ENV['RACK_ENV'] || 'development')
129
+ end
130
+ end
131
+ end
132
+ end
@@ -0,0 +1,50 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that uses multiple backends
4
+ # delegates all queries to the first backend
5
+ class Multiple < Base
6
+
7
+ class << self
8
+ attr_accessor :classes
9
+ end
10
+
11
+ def self.configure
12
+ yield self
13
+ Resque::Failure.backend = self
14
+ end
15
+
16
+ def initialize(*args)
17
+ super
18
+ @backends = self.class.classes.map {|klass| klass.new(*args)}
19
+ end
20
+
21
+ def save
22
+ @backends.each(&:save)
23
+ end
24
+
25
+ # The number of failures.
26
+ def self.count
27
+ classes.first.count
28
+ end
29
+
30
+ # Returns a paginated array of failure objects.
31
+ def self.all(start = 0, count = 1)
32
+ classes.first.all(start,count)
33
+ end
34
+
35
+ # A URL where someone can go to view failures.
36
+ def self.url
37
+ classes.first.url
38
+ end
39
+
40
+ # Clear all failure objects
41
+ def self.clear
42
+ classes.first.clear
43
+ end
44
+
45
+ def self.requeue(*args)
46
+ classes.first.requeue(*args)
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,40 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that stores exceptions in Mongo. Very simple but
4
+ # works out of the box, along with support in the Resque web app.
5
+ class Redis < Base
6
+ def save
7
+ data = {
8
+ :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
9
+ :payload => payload,
10
+ :exception => exception.class.to_s,
11
+ :error => exception.to_s,
12
+ :backtrace => Array(exception.backtrace),
13
+ :worker => worker.to_s,
14
+ :queue => queue
15
+ }
16
+ Resque.mongo_failures << data
17
+ end
18
+
19
+ def self.count
20
+ Resque.mongo_failures.count
21
+ end
22
+
23
+ def self.all(start = 0, count = 1)
24
+ all_failures = Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
25
+ # all_failures.size == 1 ? all_failures.first : all_failures
26
+ end
27
+
28
+ def self.clear
29
+ Resque.mongo_failures.remove
30
+ end
31
+
32
+ def self.requeue(index)
33
+ item = all(index)
34
+ item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
35
+ Resque.redis.lset(:failed, index, Resque.encode(item))
36
+ Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,71 @@
1
+ module Resque
2
+ # Methods used by various classes in Resque.
3
+ module Helpers
4
+ # Direct access to the Redis instance.
5
+ #def mongo
6
+ # Resque.mongo
7
+ #end
8
+
9
+ def mongo_workers
10
+ Resque.mongo_workers
11
+ end
12
+
13
+ def mongo_stats
14
+ Resque.mongo_stats
15
+ end
16
+
17
+ # Given a Ruby object, returns a string suitable for storage in a
18
+ # queue.
19
+ def encode(object)
20
+ if defined? Yajl
21
+ Yajl::Encoder.encode(object)
22
+ else
23
+ object.to_json
24
+ end
25
+ end
26
+
27
+ # Given a string, returns a Ruby object.
28
+ def decode(object)
29
+ return unless object
30
+
31
+ if defined? Yajl
32
+ begin
33
+ Yajl::Parser.parse(object, :check_utf8 => false)
34
+ rescue Yajl::ParseError
35
+ end
36
+ else
37
+ begin
38
+ JSON.parse(object)
39
+ rescue JSON::ParserError
40
+ end
41
+ end
42
+ end
43
+
44
+ # Given a word with dashes, returns a camel cased version of it.
45
+ #
46
+ # classify('job-name') # => 'JobName'
47
+ def classify(dashed_word)
48
+ dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
49
+ end
50
+
51
+ # Given a camel cased word, returns the constant it represents
52
+ #
53
+ # constantize('JobName') # => JobName
54
+ def constantize(camel_cased_word)
55
+ camel_cased_word = camel_cased_word.to_s
56
+
57
+ if camel_cased_word.include?('-')
58
+ camel_cased_word = classify(camel_cased_word)
59
+ end
60
+
61
+ names = camel_cased_word.split('::')
62
+ names.shift if names.empty? || names.first.empty?
63
+
64
+ constant = Object
65
+ names.each do |name|
66
+ constant = constant.const_get(name) || constant.const_missing(name)
67
+ end
68
+ constant
69
+ end
70
+ end
71
+ end
data/lib/resque/job.rb ADDED
@@ -0,0 +1,209 @@
1
+ module Resque
2
+ # A Resque::Job represents a unit of work. Each job lives on a
3
+ # single queue and has an associated payload object. The payload
4
+ # is a hash with two attributes: `class` and `args`. The `class` is
5
+ # the name of the Ruby class which should be used to run the
6
+ # job. The `args` are an array of arguments which should be passed
7
+ # to the Ruby class's `perform` class-level method.
8
+ #
9
+ # You can manually run a job using this code:
10
+ #
11
+ # job = Resque::Job.reserve(:high)
12
+ # klass = Resque::Job.constantize(job.payload['class'])
13
+ # klass.perform(*job.payload['args'])
14
+ class Job
15
+ include Helpers
16
+ extend Helpers
17
+
18
+ # Raise Resque::Job::DontPerform from a before_perform hook to
19
+ # abort the job.
20
+ DontPerform = Class.new(StandardError)
21
+
22
+ # The worker object which is currently processing this job.
23
+ attr_accessor :worker
24
+
25
+ # The name of the queue from which this job was pulled (or is to be
26
+ # placed)
27
+ attr_reader :queue
28
+
29
+ # This job's associated payload object.
30
+ attr_reader :payload
31
+
32
+ def initialize(queue, payload)
33
+ @queue = queue
34
+ @payload = payload
35
+
36
+ end
37
+
38
+ # Creates a job by placing it on a queue. Expects a string queue
39
+ # name, a string class name, and an optional array of arguments to
40
+ # pass to the class' `perform` method.
41
+ #
42
+ # Raises an exception if no queue or class is given.
43
+ def self.create(queue, klass, *args)
44
+ if !queue
45
+ raise NoQueueError.new("Jobs must be placed onto a queue.")
46
+ end
47
+
48
+ if klass.to_s.empty?
49
+ raise NoClassError.new("Jobs must be given a class.")
50
+ end
51
+
52
+ item = { :class => klass.to_s, :args => args}
53
+ item[:_id] = args[0][:_id] if Resque.allows_unique_jobs(klass) && args[0].is_a?(Hash) && args[0].has_key?(:_id)
54
+ item[:unique] = true if item[:_id]
55
+ item[:delay_until] = args[0][:delay_until] if Resque.allows_delayed_jobs(klass) && args[0].is_a?(Hash) && args[0].has_key?(:delay_until)
56
+
57
+
58
+ ret = Resque.push(queue, item)
59
+ Plugin.after_enqueue_hooks(klass).each do |hook|
60
+ klass.send(hook, *args)
61
+ end
62
+ ret
63
+ end
64
+
65
+ # Removes a job from a queue. Expects a string queue name, a
66
+ # string class name, and, optionally, args.
67
+ #
68
+ # Returns the number of jobs destroyed.
69
+ #
70
+ # If no args are provided, it will remove all jobs of the class
71
+ # provided.
72
+ #
73
+ # That is, for these two jobs:
74
+ #
75
+ # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
76
+ # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
77
+ #
78
+ # The following call will remove both:
79
+ #
80
+ # Resque::Job.destroy(queue, 'UpdateGraph')
81
+ #
82
+ # Whereas specifying args will only remove the 2nd job:
83
+ #
84
+ # Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
85
+ #
86
+ # This method can be potentially very slow and memory intensive,
87
+ # depending on the size of your queue, as it loads all jobs into
88
+ # a Ruby array before processing.
89
+ def self.destroy(queue, klass, *args)
90
+ collection = Resque.mongo[queue]
91
+ selector = {'class' => klass.to_s}
92
+ selector['args'] = args unless args.empty?
93
+ destroyed = collection.find(selector).count
94
+ collection.remove(selector, :safe => true)
95
+ destroyed
96
+ end
97
+
98
+ # Given a string queue name, returns an instance of Resque::Job
99
+ # if any jobs are available. If not, returns nil.
100
+ def self.reserve(queue)
101
+ return unless payload = Resque.pop(queue)
102
+ new(queue, payload)
103
+ end
104
+
105
+ # Attempts to perform the work represented by this job instance.
106
+ # Calls #perform on the class given in the payload with the
107
+ # arguments given in the payload.
108
+ def perform
109
+ job = payload_class
110
+ job_args = args || []
111
+ job_was_performed = false
112
+
113
+ before_hooks = Plugin.before_hooks(job)
114
+ around_hooks = Plugin.around_hooks(job)
115
+ after_hooks = Plugin.after_hooks(job)
116
+ failure_hooks = Plugin.failure_hooks(job)
117
+
118
+ begin
119
+ # Execute before_perform hook. Abort the job gracefully if
120
+ # Resque::DontPerform is raised.
121
+ begin
122
+ before_hooks.each do |hook|
123
+ job.send(hook, *job_args)
124
+ end
125
+ rescue DontPerform
126
+ return false
127
+ end
128
+
129
+ # Execute the job. Do it in an around_perform hook if available.
130
+ if around_hooks.empty?
131
+ job.perform(*job_args)
132
+ job_was_performed = true
133
+ else
134
+ # We want to nest all around_perform plugins, with the last one
135
+ # finally calling perform
136
+ stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
137
+ if last_hook
138
+ lambda do
139
+ job.send(hook, *job_args) { last_hook.call }
140
+ end
141
+ else
142
+ lambda do
143
+ job.send(hook, *job_args) do
144
+ result = job.perform(*job_args)
145
+ job_was_performed = true
146
+ result
147
+ end
148
+ end
149
+ end
150
+ end
151
+ stack.call
152
+ end
153
+
154
+ # Execute after_perform hook
155
+ after_hooks.each do |hook|
156
+ job.send(hook, *job_args)
157
+ end
158
+
159
+ # Return true if the job was performed
160
+ return job_was_performed
161
+
162
+ # If an exception occurs during the job execution, look for an
163
+ # on_failure hook then re-raise.
164
+ rescue Object => e
165
+ failure_hooks.each { |hook| job.send(hook, e, *job_args) }
166
+ raise e
167
+ end
168
+ end
169
+
170
+ # Returns the actual class constant represented in this job's payload.
171
+ def payload_class
172
+ @payload_class ||= constantize(@payload['class'])
173
+ end
174
+
175
+ # Returns an array of args represented in this job's payload.
176
+ def args
177
+ @payload['args']
178
+ end
179
+
180
+ # Given an exception object, hands off the needed parameters to
181
+ # the Failure module.
182
+ def fail(exception)
183
+ Failure.create \
184
+ :payload => payload,
185
+ :exception => exception,
186
+ :worker => worker,
187
+ :queue => queue
188
+ end
189
+
190
+ # Creates an identical job, essentially placing this job back on
191
+ # the queue.
192
+ def recreate
193
+ self.class.create(queue, payload_class, *args)
194
+ end
195
+
196
+ # String representation
197
+ def inspect
198
+ obj = @payload
199
+ "(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
200
+ end
201
+
202
+ # Equality
203
+ def ==(other)
204
+ queue == other.queue &&
205
+ payload_class == other.payload_class &&
206
+ args == other.args
207
+ end
208
+ end
209
+ end