nfo-resque-mongo 1.15.0
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY.md +259 -0
- data/LICENSE +20 -0
- data/README.markdown +828 -0
- data/Rakefile +73 -0
- data/bin/resque +75 -0
- data/bin/resque-web +23 -0
- data/lib/resque/errors.rb +10 -0
- data/lib/resque/failure/base.rb +74 -0
- data/lib/resque/failure/hoptoad.rb +139 -0
- data/lib/resque/failure/mongo.rb +92 -0
- data/lib/resque/failure/multiple.rb +60 -0
- data/lib/resque/failure.rb +82 -0
- data/lib/resque/helpers.rb +79 -0
- data/lib/resque/job.rb +228 -0
- data/lib/resque/plugin.rb +51 -0
- data/lib/resque/queue_stats.rb +58 -0
- data/lib/resque/server/public/idle.png +0 -0
- data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
- data/lib/resque/server/public/jquery.relatize_date.js +95 -0
- data/lib/resque/server/public/poll.png +0 -0
- data/lib/resque/server/public/ranger.js +73 -0
- data/lib/resque/server/public/reset.css +48 -0
- data/lib/resque/server/public/style.css +86 -0
- data/lib/resque/server/public/working.png +0 -0
- data/lib/resque/server/test_helper.rb +19 -0
- data/lib/resque/server/views/error.erb +1 -0
- data/lib/resque/server/views/failed.erb +75 -0
- data/lib/resque/server/views/key_sets.erb +19 -0
- data/lib/resque/server/views/key_string.erb +11 -0
- data/lib/resque/server/views/layout.erb +38 -0
- data/lib/resque/server/views/next_more.erb +19 -0
- data/lib/resque/server/views/overview.erb +4 -0
- data/lib/resque/server/views/queues.erb +49 -0
- data/lib/resque/server/views/stats.erb +62 -0
- data/lib/resque/server/views/workers.erb +109 -0
- data/lib/resque/server/views/working.erb +68 -0
- data/lib/resque/server.rb +222 -0
- data/lib/resque/stat.rb +55 -0
- data/lib/resque/tasks.rb +42 -0
- data/lib/resque/version.rb +3 -0
- data/lib/resque/worker.rb +524 -0
- data/lib/resque.rb +384 -0
- data/tasks/redis.rake +161 -0
- data/tasks/resque.rake +2 -0
- data/test/dump.rdb +0 -0
- data/test/job_hooks_test.rb +323 -0
- data/test/job_plugins_test.rb +230 -0
- data/test/plugin_test.rb +116 -0
- data/test/queue_stats_test.rb +57 -0
- data/test/redis-test.conf +115 -0
- data/test/resque-web_test.rb +48 -0
- data/test/resque_test.rb +256 -0
- data/test/test_helper.rb +151 -0
- data/test/worker_test.rb +356 -0
- metadata +166 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# A Failure backend that uses multiple backends
|
4
|
+
# delegates all queries to the first backend
|
5
|
+
class Multiple < Base
|
6
|
+
|
7
|
+
class << self
|
8
|
+
attr_accessor :classes
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.configure
|
12
|
+
yield self
|
13
|
+
Resque::Failure.backend = self
|
14
|
+
end
|
15
|
+
|
16
|
+
def initialize(*args)
|
17
|
+
super
|
18
|
+
@backends = self.class.classes.map {|klass| klass.new(*args)}
|
19
|
+
end
|
20
|
+
|
21
|
+
def save
|
22
|
+
@backends.each(&:save)
|
23
|
+
end
|
24
|
+
|
25
|
+
# The number of failures.
|
26
|
+
def self.count
|
27
|
+
classes.first.count
|
28
|
+
end
|
29
|
+
|
30
|
+
# Number of result of a failures search.
|
31
|
+
def self.search_count
|
32
|
+
classes.first.search_count if classes.first.respond_to?(:search_count)
|
33
|
+
end
|
34
|
+
|
35
|
+
# Returns a paginated array of failure objects.
|
36
|
+
def self.all(start = 0, count = 1)
|
37
|
+
classes.first.all(start,count)
|
38
|
+
end
|
39
|
+
|
40
|
+
# The results of a failures search.
|
41
|
+
def self.search_results(query, start = 0, count = 1)
|
42
|
+
classes.first.search_results(query, start, count) if classes.first.respond_to?(:search_results)
|
43
|
+
end
|
44
|
+
|
45
|
+
# A URL where someone can go to view failures.
|
46
|
+
def self.url
|
47
|
+
classes.first.url
|
48
|
+
end
|
49
|
+
|
50
|
+
# Clear all failure objects
|
51
|
+
def self.clear
|
52
|
+
classes.first.clear
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.requeue(*args)
|
56
|
+
classes.first.requeue(*args)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module Resque
|
2
|
+
# The Failure module provides an interface for working with different
|
3
|
+
# failure backends.
|
4
|
+
#
|
5
|
+
# You can use it to query the failure backend without knowing which specific
|
6
|
+
# backend is being used. For instance, the Resque web app uses it to display
|
7
|
+
# stats and other information.
|
8
|
+
module Failure
|
9
|
+
# Creates a new failure, which is delegated to the appropriate backend.
|
10
|
+
#
|
11
|
+
# Expects a hash with the following keys:
|
12
|
+
# :exception - The Exception object
|
13
|
+
# :worker - The Worker object who is reporting the failure
|
14
|
+
# :queue - The string name of the queue from which the job was pulled
|
15
|
+
# :payload - The job's payload
|
16
|
+
def self.create(options = {})
|
17
|
+
backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Sets the current backend. Expects a class descendent of
|
22
|
+
# `Resque::Failure::Base`.
|
23
|
+
#
|
24
|
+
# Example use:
|
25
|
+
# require 'resque/failure/hoptoad'
|
26
|
+
# Resque::Failure.backend = Resque::Failure::Hoptoad
|
27
|
+
def self.backend=(backend)
|
28
|
+
@backend = backend
|
29
|
+
end
|
30
|
+
|
31
|
+
# Returns the current backend class. If none has been set, falls
|
32
|
+
# back to `Resque::Failure::Redis`
|
33
|
+
def self.backend
|
34
|
+
return @backend if @backend
|
35
|
+
require 'resque/failure/mongo'
|
36
|
+
@backend = Failure::Mongo
|
37
|
+
end
|
38
|
+
|
39
|
+
# Returns the int count of how many failures we have seen.
|
40
|
+
def self.count
|
41
|
+
backend.count
|
42
|
+
end
|
43
|
+
|
44
|
+
# Returns the int count of the number of results for the last search executed
|
45
|
+
def self.search_count
|
46
|
+
backend.search_count
|
47
|
+
end
|
48
|
+
|
49
|
+
# Returns an array of all the failures, paginated.
|
50
|
+
#
|
51
|
+
# `start` is the int of the first item in the page, `count` is the
|
52
|
+
# number of items to return.
|
53
|
+
def self.all(start = 0, count = 1)
|
54
|
+
backend.all(start, count)
|
55
|
+
end
|
56
|
+
|
57
|
+
# Will hold the last count - start results of
|
58
|
+
# a search through all the failed jobs,
|
59
|
+
# using the query string 'squery'
|
60
|
+
def self.search_results(start = 0, count = 1, squery)
|
61
|
+
backend.search_results(start, count, squery)
|
62
|
+
end
|
63
|
+
|
64
|
+
# The string url of the backend's web interface, if any.
|
65
|
+
def self.url
|
66
|
+
backend.url
|
67
|
+
end
|
68
|
+
|
69
|
+
# Clear all failure jobs
|
70
|
+
def self.clear
|
71
|
+
backend.clear
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.requeue(index)
|
75
|
+
backend.requeue(index)
|
76
|
+
end
|
77
|
+
|
78
|
+
def self.remove(index)
|
79
|
+
backend.remove(index)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module Resque
|
2
|
+
# Methods used by various classes in Resque.
|
3
|
+
module Helpers
|
4
|
+
class DecodeException < StandardError; end
|
5
|
+
|
6
|
+
# Direct access to the Redis instance.
|
7
|
+
def mongo
|
8
|
+
Resque.mongo
|
9
|
+
end
|
10
|
+
|
11
|
+
def mongo_workers
|
12
|
+
Resque.mongo_workers
|
13
|
+
end
|
14
|
+
|
15
|
+
def mongo_stats
|
16
|
+
Resque.mongo_stats
|
17
|
+
end
|
18
|
+
|
19
|
+
def mongo_queues
|
20
|
+
Resque.mongo_queues
|
21
|
+
end
|
22
|
+
|
23
|
+
# Given a Ruby object, returns a string suitable for storage in a
|
24
|
+
# queue.
|
25
|
+
def encode(object)
|
26
|
+
if defined? Yajl
|
27
|
+
Yajl::Encoder.encode(object)
|
28
|
+
else
|
29
|
+
object.to_json
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Given a string, returns a Ruby object.
|
34
|
+
def decode(object)
|
35
|
+
return unless object
|
36
|
+
|
37
|
+
if defined? Yajl
|
38
|
+
begin
|
39
|
+
Yajl::Parser.parse(object, :check_utf8 => false)
|
40
|
+
rescue Yajl::ParseError => e
|
41
|
+
raise DecodeException, e
|
42
|
+
end
|
43
|
+
else
|
44
|
+
begin
|
45
|
+
JSON.parse(object)
|
46
|
+
rescue JSON::ParserError => e
|
47
|
+
raise DecodeException, e
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Given a word with dashes, returns a camel cased version of it.
|
53
|
+
#
|
54
|
+
# classify('job-name') # => 'JobName'
|
55
|
+
def classify(dashed_word)
|
56
|
+
dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
|
57
|
+
end
|
58
|
+
|
59
|
+
# Given a camel cased word, returns the constant it represents
|
60
|
+
#
|
61
|
+
# constantize('JobName') # => JobName
|
62
|
+
def constantize(camel_cased_word)
|
63
|
+
camel_cased_word = camel_cased_word.to_s
|
64
|
+
|
65
|
+
if camel_cased_word.include?('-')
|
66
|
+
camel_cased_word = classify(camel_cased_word)
|
67
|
+
end
|
68
|
+
|
69
|
+
names = camel_cased_word.split('::')
|
70
|
+
names.shift if names.empty? || names.first.empty?
|
71
|
+
|
72
|
+
constant = Object
|
73
|
+
names.each do |name|
|
74
|
+
constant = constant.const_get(name) || constant.const_missing(name)
|
75
|
+
end
|
76
|
+
constant
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
data/lib/resque/job.rb
ADDED
@@ -0,0 +1,228 @@
|
|
1
|
+
module Resque
|
2
|
+
# A Resque::Job represents a unit of work. Each job lives on a
|
3
|
+
# single queue and has an associated payload object. The payload
|
4
|
+
# is a hash with two attributes: `class` and `args`. The `class` is
|
5
|
+
# the name of the Ruby class which should be used to run the
|
6
|
+
# job. The `args` are an array of arguments which should be passed
|
7
|
+
# to the Ruby class's `perform` class-level method.
|
8
|
+
#
|
9
|
+
# You can manually run a job using this code:
|
10
|
+
#
|
11
|
+
# job = Resque::Job.reserve(:high)
|
12
|
+
# klass = Resque::Job.constantize(job.payload['class'])
|
13
|
+
# klass.perform(*job.payload['args'])
|
14
|
+
class Job
|
15
|
+
include Helpers
|
16
|
+
extend Helpers
|
17
|
+
|
18
|
+
# Raise Resque::Job::DontPerform from a before_perform hook to
|
19
|
+
# abort the job.
|
20
|
+
DontPerform = Class.new(StandardError)
|
21
|
+
|
22
|
+
# The worker object which is currently processing this job.
|
23
|
+
attr_accessor :worker
|
24
|
+
|
25
|
+
# The name of the queue from which this job was pulled (or is to be
|
26
|
+
# placed)
|
27
|
+
attr_reader :queue
|
28
|
+
|
29
|
+
# This job's associated payload object.
|
30
|
+
attr_reader :payload
|
31
|
+
|
32
|
+
attr_reader :start
|
33
|
+
|
34
|
+
def initialize(queue, payload, start = Time.now)
|
35
|
+
@queue = queue
|
36
|
+
@payload = payload
|
37
|
+
@start = start
|
38
|
+
end
|
39
|
+
|
40
|
+
# Creates a job by placing it on a queue. Expects a string queue
|
41
|
+
# name, a string class name, and an optional array of arguments to
|
42
|
+
# pass to the class' `perform` method.
|
43
|
+
#
|
44
|
+
# Raises an exception if no queue or class is given.
|
45
|
+
def self.create(queue, klass, *args)
|
46
|
+
Resque.validate(klass, queue)
|
47
|
+
|
48
|
+
if Resque.inline?
|
49
|
+
constantize(klass).perform(*decode(encode(args)))
|
50
|
+
else
|
51
|
+
Resque.push(queue, :class => klass.to_s, :args => args)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.from_hash(hash)
|
56
|
+
new(hash['queue'],hash['payload'],Time.parse(hash['run_at']))
|
57
|
+
end
|
58
|
+
|
59
|
+
# Removes a job from a queue. Expects a string queue name, a
|
60
|
+
# string class name, and, optionally, args.
|
61
|
+
#
|
62
|
+
# Returns the number of jobs destroyed.
|
63
|
+
#
|
64
|
+
# If no args are provided, it will remove all jobs of the class
|
65
|
+
# provided.
|
66
|
+
#
|
67
|
+
# That is, for these two jobs:
|
68
|
+
#
|
69
|
+
# { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
|
70
|
+
# { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
|
71
|
+
#
|
72
|
+
# The following call will remove both:
|
73
|
+
#
|
74
|
+
# Resque::Job.destroy(queue, 'UpdateGraph')
|
75
|
+
#
|
76
|
+
# Whereas specifying args will only remove the 2nd job:
|
77
|
+
#
|
78
|
+
# Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
|
79
|
+
#
|
80
|
+
# This method can be potentially very slow and memory intensive,
|
81
|
+
# depending on the size of your queue, as it loads all jobs into
|
82
|
+
# a Ruby array before processing.
|
83
|
+
def self.destroy(queue, klass, *args)
|
84
|
+
klass = klass.to_s
|
85
|
+
|
86
|
+
destroyed = 0
|
87
|
+
|
88
|
+
### find the item using a mongo query, that's essentially why we removed decode / encode
|
89
|
+
### for the job payload. This should be much faster than loading every job and matching
|
90
|
+
### them against our conditions !
|
91
|
+
|
92
|
+
mongo_query = { :queue => queue.to_s, 'item.class' => klass }
|
93
|
+
unless args.empty?
|
94
|
+
mongo_query.merge!({ 'item.args' => args })
|
95
|
+
end
|
96
|
+
job_count = mongo.find(mongo_query).count
|
97
|
+
mongo.remove(mongo_query, :safe => true)
|
98
|
+
QueueStats.remove_job(queue,job_count)
|
99
|
+
job_count
|
100
|
+
|
101
|
+
# mongo.find(mongo_query).each do |rec|
|
102
|
+
# json = decode(rec['item'])
|
103
|
+
#
|
104
|
+
# match = json['class'] == klass
|
105
|
+
# match &= json['args'] == args unless args.empty?
|
106
|
+
#
|
107
|
+
# if match
|
108
|
+
# destroyed += 1
|
109
|
+
# mongo.remove(:_id => rec['_id'])
|
110
|
+
# end
|
111
|
+
# end
|
112
|
+
#
|
113
|
+
# destroyed
|
114
|
+
end
|
115
|
+
|
116
|
+
# Given a string queue name, returns an instance of Resque::Job
|
117
|
+
# if any jobs are available. If not, returns nil.
|
118
|
+
def self.reserve(queue)
|
119
|
+
return unless payload = Resque.pop(queue)
|
120
|
+
new(queue, payload)
|
121
|
+
end
|
122
|
+
|
123
|
+
# Attempts to perform the work represented by this job instance.
|
124
|
+
# Calls #perform on the class given in the payload with the
|
125
|
+
# arguments given in the payload.
|
126
|
+
def perform
|
127
|
+
job = payload_class
|
128
|
+
job_args = args || []
|
129
|
+
job_was_performed = false
|
130
|
+
|
131
|
+
before_hooks = Plugin.before_hooks(job)
|
132
|
+
around_hooks = Plugin.around_hooks(job)
|
133
|
+
after_hooks = Plugin.after_hooks(job)
|
134
|
+
failure_hooks = Plugin.failure_hooks(job)
|
135
|
+
|
136
|
+
begin
|
137
|
+
# Execute before_perform hook. Abort the job gracefully if
|
138
|
+
# Resque::DontPerform is raised.
|
139
|
+
begin
|
140
|
+
before_hooks.each do |hook|
|
141
|
+
job.send(hook, *job_args)
|
142
|
+
end
|
143
|
+
rescue DontPerform
|
144
|
+
return false
|
145
|
+
end
|
146
|
+
|
147
|
+
# Execute the job. Do it in an around_perform hook if available.
|
148
|
+
if around_hooks.empty?
|
149
|
+
job.perform(*job_args)
|
150
|
+
job_was_performed = true
|
151
|
+
else
|
152
|
+
# We want to nest all around_perform plugins, with the last one
|
153
|
+
# finally calling perform
|
154
|
+
stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
|
155
|
+
if last_hook
|
156
|
+
lambda do
|
157
|
+
job.send(hook, *job_args) { last_hook.call }
|
158
|
+
end
|
159
|
+
else
|
160
|
+
lambda do
|
161
|
+
job.send(hook, *job_args) do
|
162
|
+
result = job.perform(*job_args)
|
163
|
+
job_was_performed = true
|
164
|
+
result
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
stack.call
|
170
|
+
end
|
171
|
+
|
172
|
+
# Execute after_perform hook
|
173
|
+
after_hooks.each do |hook|
|
174
|
+
job.send(hook, *job_args)
|
175
|
+
end
|
176
|
+
|
177
|
+
# Return true if the job was performed
|
178
|
+
return job_was_performed
|
179
|
+
|
180
|
+
# If an exception occurs during the job execution, look for an
|
181
|
+
# on_failure hook then re-raise.
|
182
|
+
rescue Object => e
|
183
|
+
failure_hooks.each { |hook| job.send(hook, e, *job_args) }
|
184
|
+
raise e
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
# Returns the actual class constant represented in this job's payload.
|
189
|
+
def payload_class
|
190
|
+
@payload_class ||= constantize(@payload['class'])
|
191
|
+
end
|
192
|
+
|
193
|
+
# Returns an array of args represented in this job's payload.
|
194
|
+
def args
|
195
|
+
@payload['args']
|
196
|
+
end
|
197
|
+
|
198
|
+
# Given an exception object, hands off the needed parameters to
|
199
|
+
# the Failure module.
|
200
|
+
def fail(exception)
|
201
|
+
Failure.create \
|
202
|
+
:payload => payload,
|
203
|
+
:exception => exception,
|
204
|
+
:worker => worker,
|
205
|
+
:queue => queue
|
206
|
+
end
|
207
|
+
|
208
|
+
# Creates an identical job, essentially placing this job back on
|
209
|
+
# the queue.
|
210
|
+
def recreate
|
211
|
+
self.class.create(queue, payload_class, *args)
|
212
|
+
end
|
213
|
+
|
214
|
+
# String representation
|
215
|
+
def inspect
|
216
|
+
obj = @payload
|
217
|
+
"(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
|
218
|
+
end
|
219
|
+
|
220
|
+
# Equality
|
221
|
+
def ==(other)
|
222
|
+
queue == other.queue &&
|
223
|
+
payload_class == other.payload_class &&
|
224
|
+
args == other.args
|
225
|
+
end
|
226
|
+
|
227
|
+
end
|
228
|
+
end
|