resque-mongo 1.3.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -0
- data/.kick +26 -0
- data/CONTRIBUTORS +14 -0
- data/HISTORY.md +53 -0
- data/LICENSE +20 -0
- data/README.markdown +755 -0
- data/Rakefile +65 -0
- data/bin/resque +57 -0
- data/bin/resque-web +18 -0
- data/config.ru +14 -0
- data/deps.rip +8 -0
- data/examples/async_helper.rb +31 -0
- data/examples/demo/README.markdown +71 -0
- data/examples/demo/Rakefile +3 -0
- data/examples/demo/app.rb +27 -0
- data/examples/demo/config.ru +19 -0
- data/examples/demo/job.rb +12 -0
- data/examples/god/resque.god +52 -0
- data/examples/god/stale.god +26 -0
- data/examples/instance.rb +11 -0
- data/examples/simple.rb +30 -0
- data/init.rb +1 -0
- data/lib/resque.rb +224 -0
- data/lib/resque/errors.rb +7 -0
- data/lib/resque/failure.rb +63 -0
- data/lib/resque/failure/base.rb +58 -0
- data/lib/resque/failure/hoptoad.rb +88 -0
- data/lib/resque/failure/mongo.rb +32 -0
- data/lib/resque/helpers.rb +65 -0
- data/lib/resque/job.rb +103 -0
- data/lib/resque/server.rb +182 -0
- data/lib/resque/server/public/idle.png +0 -0
- data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
- data/lib/resque/server/public/jquery.relatize_date.js +95 -0
- data/lib/resque/server/public/poll.png +0 -0
- data/lib/resque/server/public/ranger.js +24 -0
- data/lib/resque/server/public/reset.css +48 -0
- data/lib/resque/server/public/style.css +75 -0
- data/lib/resque/server/public/working.png +0 -0
- data/lib/resque/server/views/error.erb +1 -0
- data/lib/resque/server/views/failed.erb +35 -0
- data/lib/resque/server/views/key.erb +17 -0
- data/lib/resque/server/views/layout.erb +38 -0
- data/lib/resque/server/views/next_more.erb +10 -0
- data/lib/resque/server/views/overview.erb +4 -0
- data/lib/resque/server/views/queues.erb +46 -0
- data/lib/resque/server/views/stats.erb +62 -0
- data/lib/resque/server/views/workers.erb +78 -0
- data/lib/resque/server/views/working.erb +67 -0
- data/lib/resque/stat.rb +55 -0
- data/lib/resque/tasks.rb +39 -0
- data/lib/resque/version.rb +3 -0
- data/lib/resque/worker.rb +415 -0
- data/tasks/redis.rake +133 -0
- data/tasks/resque.rake +2 -0
- data/test/dump.rdb +0 -0
- data/test/redis-test.conf +132 -0
- data/test/resque_test.rb +191 -0
- data/test/test_helper.rb +87 -0
- data/test/worker_test.rb +229 -0
- metadata +162 -0
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'resque'
|
data/lib/resque.rb
ADDED
@@ -0,0 +1,224 @@
|
|
1
|
+
require 'mongo'
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'yajl'
|
5
|
+
rescue LoadError
|
6
|
+
require 'json'
|
7
|
+
end
|
8
|
+
|
9
|
+
require 'resque/errors'
|
10
|
+
|
11
|
+
require 'resque/failure'
|
12
|
+
require 'resque/failure/base'
|
13
|
+
|
14
|
+
require 'resque/helpers'
|
15
|
+
require 'resque/stat'
|
16
|
+
require 'resque/job'
|
17
|
+
require 'resque/worker'
|
18
|
+
|
19
|
+
module Resque
|
20
|
+
include Helpers
|
21
|
+
extend self
|
22
|
+
|
23
|
+
# Accepts a 'hostname:port' string or a Redis server.
|
24
|
+
def mongo=(server)
|
25
|
+
case server
|
26
|
+
when String
|
27
|
+
host, port = server.split(':')
|
28
|
+
@con = Mongo::Connection.new(host, port)
|
29
|
+
@db = @con.db('monque')
|
30
|
+
@mongo = @db.collection('monque')
|
31
|
+
@workers = @db.collection('workers')
|
32
|
+
@failures = @db.collection('failures')
|
33
|
+
@stats = @db.collection('stats')
|
34
|
+
|
35
|
+
add_indexes
|
36
|
+
else
|
37
|
+
raise "I don't know what to do with #{server.inspect}"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
# Returns the current Redis connection. If none has been created, will
|
43
|
+
# create a new one.
|
44
|
+
def mongo
|
45
|
+
return @mongo if @mongo
|
46
|
+
self.mongo = 'localhost:27017'
|
47
|
+
self.mongo
|
48
|
+
end
|
49
|
+
|
50
|
+
def mongo_workers
|
51
|
+
return @workers if @workers
|
52
|
+
self.mongo = 'localhost:27017'
|
53
|
+
@workers
|
54
|
+
end
|
55
|
+
|
56
|
+
def mongo_failures
|
57
|
+
return @failures if @failures
|
58
|
+
self.mongo = 'localhost:27017'
|
59
|
+
@failures
|
60
|
+
end
|
61
|
+
|
62
|
+
def mongo_stats
|
63
|
+
return @stats if @stats
|
64
|
+
self.mongo = 'localhost:27017'
|
65
|
+
@stats
|
66
|
+
end
|
67
|
+
|
68
|
+
def to_s
|
69
|
+
"Mongo Client connected to #{@con.host}"
|
70
|
+
end
|
71
|
+
|
72
|
+
def add_indexes
|
73
|
+
@mongo.create_index :queue
|
74
|
+
@workers.create_index :worker
|
75
|
+
@stats.create_index :stat
|
76
|
+
end
|
77
|
+
|
78
|
+
def drop
|
79
|
+
@mongo.drop
|
80
|
+
@workers.drop
|
81
|
+
@failures.drop
|
82
|
+
@stats.drop
|
83
|
+
@mongo = nil
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# queue manipulation
|
88
|
+
#
|
89
|
+
|
90
|
+
# Pushes a job onto a queue. Queue name should be a string and the
|
91
|
+
# item should be any JSON-able Ruby object.
|
92
|
+
def push(queue, item)
|
93
|
+
watch_queue(queue)
|
94
|
+
mongo << { :queue => queue.to_s, :item => encode(item) }
|
95
|
+
end
|
96
|
+
|
97
|
+
# Pops a job off a queue. Queue name should be a string.
|
98
|
+
#
|
99
|
+
# Returns a Ruby object.
|
100
|
+
def pop(queue)
|
101
|
+
doc = mongo.find_modify( :query => { :queue => queue },
|
102
|
+
:sort => [:natural, :desc],
|
103
|
+
:remove => true )
|
104
|
+
decode doc['item']
|
105
|
+
rescue Mongo::OperationFailure => e
|
106
|
+
return nil if e.message =~ /No matching object/
|
107
|
+
raise e
|
108
|
+
end
|
109
|
+
|
110
|
+
# Returns an int representing the size of a queue.
|
111
|
+
# Queue name should be a string.
|
112
|
+
def size(queue)
|
113
|
+
mongo.find(:queue => queue).count
|
114
|
+
end
|
115
|
+
|
116
|
+
# Returns an array of items currently queued. Queue name should be
|
117
|
+
# a string.
|
118
|
+
#
|
119
|
+
# start and count should be integer and can be used for pagination.
|
120
|
+
# start is the item to begin, count is how many items to return.
|
121
|
+
#
|
122
|
+
# To get the 3rd page of a 30 item, paginatied list one would use:
|
123
|
+
# Resque.peek('my_list', 59, 30)
|
124
|
+
def peek(queue, start = 0, count = 1)
|
125
|
+
res = mongo.find(:queue => queue).sort([:natural, :desc]).skip(start).limit(count).to_a
|
126
|
+
res.collect! { |doc| decode(doc['item']) }
|
127
|
+
|
128
|
+
if count == 1
|
129
|
+
return nil if res.empty?
|
130
|
+
res.first
|
131
|
+
else
|
132
|
+
return [] if res.empty?
|
133
|
+
res
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Returns an array of all known Resque queues as strings.
|
138
|
+
def queues
|
139
|
+
mongo.distinct(:queue)
|
140
|
+
end
|
141
|
+
|
142
|
+
# Given a queue name, completely deletes the queue.
|
143
|
+
def remove_queue(queue)
|
144
|
+
mongo.remove(:queue => queue)
|
145
|
+
end
|
146
|
+
|
147
|
+
# Used internally to keep track of which queues we've created.
|
148
|
+
# Don't call this directly.
|
149
|
+
def watch_queue(queue)
|
150
|
+
# redis.sadd(:queues, queue.to_s)
|
151
|
+
end
|
152
|
+
|
153
|
+
|
154
|
+
#
|
155
|
+
# job shortcuts
|
156
|
+
#
|
157
|
+
|
158
|
+
# This method can be used to conveniently add a job to a queue.
|
159
|
+
# It assumes the class you're passing it is a real Ruby class (not
|
160
|
+
# a string or reference) which either:
|
161
|
+
#
|
162
|
+
# a) has a @queue ivar set
|
163
|
+
# b) responds to `queue`
|
164
|
+
#
|
165
|
+
# If either of those conditions are met, it will use the value obtained
|
166
|
+
# from performing one of the above operations to determine the queue.
|
167
|
+
#
|
168
|
+
# If no queue can be inferred this method will return a non-true value.
|
169
|
+
#
|
170
|
+
# This method is considered part of the `stable` API.
|
171
|
+
def enqueue(klass, *args)
|
172
|
+
queue = klass.instance_variable_get(:@queue)
|
173
|
+
queue ||= klass.queue if klass.respond_to?(:queue)
|
174
|
+
Job.create(queue, klass, *args)
|
175
|
+
end
|
176
|
+
|
177
|
+
# This method will return a `Resque::Job` object or a non-true value
|
178
|
+
# depending on whether a job can be obtained. You should pass it the
|
179
|
+
# precise name of a queue: case matters.
|
180
|
+
#
|
181
|
+
# This method is considered part of the `stable` API.
|
182
|
+
def reserve(queue)
|
183
|
+
Job.reserve(queue)
|
184
|
+
end
|
185
|
+
|
186
|
+
|
187
|
+
#
|
188
|
+
# worker shortcuts
|
189
|
+
#
|
190
|
+
|
191
|
+
# A shortcut to Worker.all
|
192
|
+
def workers
|
193
|
+
Worker.all
|
194
|
+
end
|
195
|
+
|
196
|
+
# A shortcut to Worker.working
|
197
|
+
def working
|
198
|
+
Worker.working
|
199
|
+
end
|
200
|
+
|
201
|
+
|
202
|
+
#
|
203
|
+
# stats
|
204
|
+
#
|
205
|
+
|
206
|
+
# Returns a hash, similar to redis-rb's #info, of interesting stats.
|
207
|
+
def info
|
208
|
+
return {
|
209
|
+
:pending => queues.inject(0) { |m,k| m + size(k) },
|
210
|
+
:processed => Stat[:processed],
|
211
|
+
:queues => queues.size,
|
212
|
+
:workers => workers.size.to_i,
|
213
|
+
:working => working.size,
|
214
|
+
:failed => Stat[:failed],
|
215
|
+
:servers => ["#{@con.host}:#{@con.port}"]
|
216
|
+
}
|
217
|
+
end
|
218
|
+
|
219
|
+
# Returns an array of all known Resque keys in Redis. Redis' KEYS operation
|
220
|
+
# is O(N) for the keyspace, so be careful - this can be slow for big databases.
|
221
|
+
def keys
|
222
|
+
queues
|
223
|
+
end
|
224
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module Resque
|
2
|
+
# The Failure module provides an interface for working with different
|
3
|
+
# failure backends.
|
4
|
+
#
|
5
|
+
# You can use it to query the failure backend without knowing which specific
|
6
|
+
# backend is being used. For instance, the Resque web app uses it to display
|
7
|
+
# stats and other information.
|
8
|
+
module Failure
|
9
|
+
# Creates a new failure, which is delegated to the appropriate backend.
|
10
|
+
#
|
11
|
+
# Expects a hash with the following keys:
|
12
|
+
# :exception - The Exception object
|
13
|
+
# :worker - The Worker object who is reporting the failure
|
14
|
+
# :queue - The string name of the queue from which the job was pulled
|
15
|
+
# :payload - The job's payload
|
16
|
+
def self.create(options = {})
|
17
|
+
backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Sets the current backend. Expects a class descendent of
|
22
|
+
# `Resque::Failure::Base`.
|
23
|
+
#
|
24
|
+
# Example use:
|
25
|
+
# require 'resque/failure/hoptoad'
|
26
|
+
# Resque::Failure.backend = Resque::Failure::Hoptoad
|
27
|
+
def self.backend=(backend)
|
28
|
+
@backend = backend
|
29
|
+
end
|
30
|
+
|
31
|
+
# Returns the current backend class. If none has been set, falls
|
32
|
+
# back to `Resque::Failure::Redis`
|
33
|
+
def self.backend
|
34
|
+
return @backend if @backend
|
35
|
+
require 'resque/failure/mongo'
|
36
|
+
@backend = Failure::Mongo
|
37
|
+
end
|
38
|
+
|
39
|
+
# Returns the int count of how many failures we have seen.
|
40
|
+
def self.count
|
41
|
+
backend.count
|
42
|
+
end
|
43
|
+
|
44
|
+
# Returns an array of all the failures, paginated.
|
45
|
+
#
|
46
|
+
# `start` is the int of the first item in the page, `count` is the
|
47
|
+
# number of items to return.
|
48
|
+
def self.all(start = 0, count = 1)
|
49
|
+
backend.all(start, count)
|
50
|
+
end
|
51
|
+
|
52
|
+
# The string url of the backend's web interface, if any.
|
53
|
+
def self.url
|
54
|
+
backend.url
|
55
|
+
end
|
56
|
+
|
57
|
+
# Clear all failure jobs
|
58
|
+
def self.clear
|
59
|
+
backend.clear
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# All Failure classes are expected to subclass Base.
|
4
|
+
#
|
5
|
+
# When a job fails, a new instance of your Failure backend is created
|
6
|
+
# and #save is called.
|
7
|
+
class Base
|
8
|
+
# The exception object raised by the failed job
|
9
|
+
attr_accessor :exception
|
10
|
+
|
11
|
+
# The worker object who detected the failure
|
12
|
+
attr_accessor :worker
|
13
|
+
|
14
|
+
# The string name of the queue from which the failed job was pulled
|
15
|
+
attr_accessor :queue
|
16
|
+
|
17
|
+
# The payload object associated with the failed job
|
18
|
+
attr_accessor :payload
|
19
|
+
|
20
|
+
def initialize(exception, worker, queue, payload)
|
21
|
+
@exception = exception
|
22
|
+
@worker = worker
|
23
|
+
@queue = queue
|
24
|
+
@payload = payload
|
25
|
+
end
|
26
|
+
|
27
|
+
# When a job fails, a new instance of your Failure backend is created
|
28
|
+
# and #save is called.
|
29
|
+
#
|
30
|
+
# This is where you POST or PUT or whatever to your Failure service.
|
31
|
+
def save
|
32
|
+
end
|
33
|
+
|
34
|
+
# The number of failures.
|
35
|
+
def self.count
|
36
|
+
0
|
37
|
+
end
|
38
|
+
|
39
|
+
# Returns a paginated array of failure objects.
|
40
|
+
def self.all(start = 0, count = 1)
|
41
|
+
[]
|
42
|
+
end
|
43
|
+
|
44
|
+
# A URL where someone can go to view failures.
|
45
|
+
def self.url
|
46
|
+
end
|
47
|
+
|
48
|
+
# Clear all failure objects
|
49
|
+
def self.clear
|
50
|
+
end
|
51
|
+
|
52
|
+
# Logging!
|
53
|
+
def log(message)
|
54
|
+
@worker.log(message)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Failure
|
5
|
+
# A Failure backend that sends exceptions raised by jobs to Hoptoad.
|
6
|
+
#
|
7
|
+
# To use it, put this code in an initializer, Rake task, or wherever:
|
8
|
+
#
|
9
|
+
# Resque::Failure::Hoptoad.configure do |config|
|
10
|
+
# config.api_key = 'blah'
|
11
|
+
# config.secure = true
|
12
|
+
# config.subdomain = 'your_hoptoad_subdomain'
|
13
|
+
# end
|
14
|
+
class Hoptoad < Base
|
15
|
+
class << self
|
16
|
+
attr_accessor :secure, :api_key, :subdomain
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.url
|
20
|
+
"http://#{subdomain}.hoptoadapp.com/" if subdomain
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.count
|
24
|
+
# We can't get the total # of errors from Hoptoad so we fake it
|
25
|
+
# by asking Resque how many errors it has seen.
|
26
|
+
Stat[:failed]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.configure
|
30
|
+
yield self
|
31
|
+
Resque::Failure.backend = self
|
32
|
+
end
|
33
|
+
|
34
|
+
def save
|
35
|
+
data = {
|
36
|
+
:api_key => api_key,
|
37
|
+
:error_class => exception.class.name,
|
38
|
+
:error_message => "#{exception.class.name}: #{exception.message}",
|
39
|
+
:backtrace => exception.backtrace,
|
40
|
+
:environment => {},
|
41
|
+
:session => {},
|
42
|
+
:request => {
|
43
|
+
:params => payload.merge(:worker => worker.to_s, :queue => queue.to_s)
|
44
|
+
}
|
45
|
+
}
|
46
|
+
|
47
|
+
send_to_hoptoad(:notice => data)
|
48
|
+
end
|
49
|
+
|
50
|
+
def send_to_hoptoad(data)
|
51
|
+
http = use_ssl? ? :https : :http
|
52
|
+
url = URI.parse("#{http}://hoptoadapp.com/notices/")
|
53
|
+
|
54
|
+
http = Net::HTTP.new(url.host, url.port)
|
55
|
+
headers = {
|
56
|
+
'Content-type' => 'application/json',
|
57
|
+
'Accept' => 'text/xml, application/xml'
|
58
|
+
}
|
59
|
+
|
60
|
+
http.read_timeout = 5 # seconds
|
61
|
+
http.open_timeout = 2 # seconds
|
62
|
+
http.use_ssl = use_ssl?
|
63
|
+
|
64
|
+
begin
|
65
|
+
response = http.post(url.path, Resque.encode(data), headers)
|
66
|
+
rescue TimeoutError => e
|
67
|
+
log "Timeout while contacting the Hoptoad server."
|
68
|
+
end
|
69
|
+
|
70
|
+
case response
|
71
|
+
when Net::HTTPSuccess then
|
72
|
+
log "Hoptoad Success: #{response.class}"
|
73
|
+
else
|
74
|
+
body = response.body if response.respond_to? :body
|
75
|
+
log "Hoptoad Failure: #{response.class}\n#{body}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def use_ssl?
|
80
|
+
self.class.secure
|
81
|
+
end
|
82
|
+
|
83
|
+
def api_key
|
84
|
+
self.class.api_key
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# A Failure backend that stores exceptions in Mongo. Very simple but
|
4
|
+
# works out of the box, along with support in the Resque web app.
|
5
|
+
class Mongo < Base
|
6
|
+
def save
|
7
|
+
data = {
|
8
|
+
:failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
|
9
|
+
:payload => payload,
|
10
|
+
:error => exception.to_s,
|
11
|
+
:backtrace => exception.backtrace,
|
12
|
+
:worker => worker.to_s,
|
13
|
+
:queue => queue
|
14
|
+
}
|
15
|
+
Resque.mongo_failures << data
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.count
|
19
|
+
Resque.mongo_failures.count
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.all(start = 0, count = 1)
|
23
|
+
Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.clear
|
27
|
+
Resque.mongo_failures.remove
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|