resque 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of resque might be problematic. Click here for more details.
- data/.kick +26 -0
- data/HISTORY.md +3 -0
- data/LICENSE +20 -0
- data/README.markdown +638 -0
- data/Rakefile +61 -0
- data/TODO.md +60 -0
- data/bin/resque +57 -0
- data/bin/resque-web +47 -0
- data/config.ru +8 -0
- data/deps.rip +5 -0
- data/examples/async_helper.rb +31 -0
- data/examples/demo/README.markdown +71 -0
- data/examples/demo/Rakefile +3 -0
- data/examples/demo/app.rb +27 -0
- data/examples/demo/config.ru +19 -0
- data/examples/demo/job.rb +12 -0
- data/examples/existing_classes_as_jobs.rb +3 -0
- data/examples/instance.rb +11 -0
- data/examples/simple.rb +30 -0
- data/init.rb +1 -0
- data/lib/resque.rb +184 -0
- data/lib/resque/errors.rb +7 -0
- data/lib/resque/failure.rb +57 -0
- data/lib/resque/failure/base.rb +54 -0
- data/lib/resque/failure/hoptoad.rb +88 -0
- data/lib/resque/failure/redis.rb +28 -0
- data/lib/resque/helpers.rb +57 -0
- data/lib/resque/job.rb +91 -0
- data/lib/resque/server.rb +154 -0
- data/lib/resque/server/public/idle.png +0 -0
- data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
- data/lib/resque/server/public/jquery.relatize_date.js +95 -0
- data/lib/resque/server/public/nav-bg.png +0 -0
- data/lib/resque/server/public/ranger.js +7 -0
- data/lib/resque/server/public/reset.css +51 -0
- data/lib/resque/server/public/style.css +67 -0
- data/lib/resque/server/public/tab_b.gif +0 -0
- data/lib/resque/server/public/tab_r.gif +0 -0
- data/lib/resque/server/public/tabs.css +189 -0
- data/lib/resque/server/public/working.png +0 -0
- data/lib/resque/server/views/error.erb +1 -0
- data/lib/resque/server/views/failed.erb +29 -0
- data/lib/resque/server/views/key.erb +17 -0
- data/lib/resque/server/views/layout.erb +43 -0
- data/lib/resque/server/views/next_more.erb +12 -0
- data/lib/resque/server/views/overview.erb +2 -0
- data/lib/resque/server/views/queues.erb +40 -0
- data/lib/resque/server/views/stats.erb +62 -0
- data/lib/resque/server/views/workers.erb +72 -0
- data/lib/resque/server/views/working.erb +66 -0
- data/lib/resque/stat.rb +53 -0
- data/lib/resque/tasks.rb +24 -0
- data/lib/resque/version.rb +3 -0
- data/lib/resque/worker.rb +406 -0
- data/tasks/redis.rake +125 -0
- data/tasks/resque.rake +2 -0
- data/test/redis-test.conf +132 -0
- data/test/resque_test.rb +160 -0
- data/test/test_helper.rb +90 -0
- data/test/worker_test.rb +212 -0
- metadata +124 -0
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'resque'
|
data/lib/resque.rb
ADDED
@@ -0,0 +1,184 @@
|
|
1
|
+
require 'redis/namespace'
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'yajl'
|
5
|
+
rescue LoadError
|
6
|
+
require 'json'
|
7
|
+
end
|
8
|
+
|
9
|
+
require 'resque/errors'
|
10
|
+
|
11
|
+
require 'resque/failure'
|
12
|
+
require 'resque/failure/base'
|
13
|
+
|
14
|
+
require 'resque/helpers'
|
15
|
+
require 'resque/stat'
|
16
|
+
require 'resque/job'
|
17
|
+
require 'resque/worker'
|
18
|
+
|
19
|
+
module Resque
|
20
|
+
include Helpers
|
21
|
+
extend self
|
22
|
+
|
23
|
+
# Accepts a 'hostname:port' string or a Redis server.
|
24
|
+
def redis=(server)
|
25
|
+
case server
|
26
|
+
when String
|
27
|
+
host, port = server.split(':')
|
28
|
+
redis = Redis.new(:host => host, :port => port, :thread_safe => true)
|
29
|
+
@redis = Redis::Namespace.new(:resque, :redis => redis)
|
30
|
+
when Redis
|
31
|
+
@redis = Redis::Namespace.new(:resque, :redis => server)
|
32
|
+
else
|
33
|
+
raise "I don't know what to do with #{server.inspect}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Returns the current Redis connection. If none has been created, will
|
38
|
+
# create a new one.
|
39
|
+
def redis
|
40
|
+
return @redis if @redis
|
41
|
+
self.redis = 'localhost:6379'
|
42
|
+
self.redis
|
43
|
+
end
|
44
|
+
|
45
|
+
def to_s
|
46
|
+
"Resque Client connected to #{redis.server}"
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
#
|
51
|
+
# queue manipulation
|
52
|
+
#
|
53
|
+
|
54
|
+
# Pushes a job onto a queue. Queue name should be a string and the
|
55
|
+
# item should be any JSON-able Ruby object.
|
56
|
+
def push(queue, item)
|
57
|
+
watch_queue(queue)
|
58
|
+
redis.rpush "queue:#{queue}", encode(item)
|
59
|
+
end
|
60
|
+
|
61
|
+
# Pops a job off a queue. Queue name should be a string.
|
62
|
+
#
|
63
|
+
# Returns a Ruby object.
|
64
|
+
def pop(queue)
|
65
|
+
decode redis.lpop("queue:#{queue}")
|
66
|
+
end
|
67
|
+
|
68
|
+
# Returns an int representing the size of a queue.
|
69
|
+
# Queue name should be a string.
|
70
|
+
def size(queue)
|
71
|
+
redis.llen("queue:#{queue}").to_i
|
72
|
+
end
|
73
|
+
|
74
|
+
# Returns an array of items currently queued. Queue name should be
|
75
|
+
# a string.
|
76
|
+
#
|
77
|
+
# start and count should be integer and can be used for pagination.
|
78
|
+
# start is the item to begin, count is how many items to return.
|
79
|
+
#
|
80
|
+
# To get the 3rd page of a 30 item, paginatied list one would use:
|
81
|
+
# Resque.peek('my_list', 59, 30)
|
82
|
+
def peek(queue, start = 0, count = 1)
|
83
|
+
list_range("queue:#{queue}", start, count)
|
84
|
+
end
|
85
|
+
|
86
|
+
# Does the dirty work of fetching a range of items from a Redis list
|
87
|
+
# and converting them into Ruby objects.
|
88
|
+
def list_range(key, start = 0, count = 1)
|
89
|
+
if count == 1
|
90
|
+
decode redis.lindex(key, start)
|
91
|
+
else
|
92
|
+
Array(redis.lrange(key, start, start+count-1)).map do |item|
|
93
|
+
decode item
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Returns an array of all known Resque queues as strings.
|
99
|
+
def queues
|
100
|
+
redis.smembers(:queues)
|
101
|
+
end
|
102
|
+
|
103
|
+
# Used internally to keep track of which queues we've created.
|
104
|
+
# Don't call this directly.
|
105
|
+
def watch_queue(queue)
|
106
|
+
@watched_queues ||= {}
|
107
|
+
return if @watched_queues[queue]
|
108
|
+
redis.sadd(:queues, queue.to_s)
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
#
|
113
|
+
# job shortcuts
|
114
|
+
#
|
115
|
+
|
116
|
+
# This method can be used to conveniently add a job to a queue.
|
117
|
+
# It assumes the class you're passing it is a real Ruby class (not
|
118
|
+
# a string or reference) which either:
|
119
|
+
#
|
120
|
+
# a) has a @queue ivar set
|
121
|
+
# b) responds to `queue`
|
122
|
+
#
|
123
|
+
# If either of those conditions are met, it will use the value obtained
|
124
|
+
# from performing one of the above operations to determine the queue.
|
125
|
+
#
|
126
|
+
# If no queue can be inferred this method will return a non-true value.
|
127
|
+
#
|
128
|
+
# This method is considered part of the `stable` API.
|
129
|
+
def enqueue(klass, *args)
|
130
|
+
queue = klass.instance_variable_get(:@queue)
|
131
|
+
queue ||= klass.queue if klass.respond_to?(:queue)
|
132
|
+
Job.create(queue, klass, *args)
|
133
|
+
end
|
134
|
+
|
135
|
+
# This method will return a `Resque::Job` object or a non-true value
|
136
|
+
# depending on whether a job can be obtained. You should pass it the
|
137
|
+
# precise name of a queue: case matters.
|
138
|
+
#
|
139
|
+
# This method is considered part of the `stable` API.
|
140
|
+
def reserve(queue)
|
141
|
+
Job.reserve(queue)
|
142
|
+
end
|
143
|
+
|
144
|
+
|
145
|
+
#
|
146
|
+
# worker shortcuts
|
147
|
+
#
|
148
|
+
|
149
|
+
# A shortcut to Worker.all
|
150
|
+
def workers
|
151
|
+
Worker.all
|
152
|
+
end
|
153
|
+
|
154
|
+
# A shortcut to Worker.working
|
155
|
+
def working
|
156
|
+
Worker.working
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
#
|
161
|
+
# stats
|
162
|
+
#
|
163
|
+
|
164
|
+
# Returns a hash, similar to redis-rb's #info, of interesting stats.
|
165
|
+
def info
|
166
|
+
return {
|
167
|
+
:pending => queues.inject(0) { |m,k| m + size(k) },
|
168
|
+
:processed => Stat[:processed],
|
169
|
+
:queues => queues.size,
|
170
|
+
:workers => workers.size.to_i,
|
171
|
+
:working => working.size,
|
172
|
+
:failed => Stat[:failed],
|
173
|
+
:servers => [redis.server]
|
174
|
+
}
|
175
|
+
end
|
176
|
+
|
177
|
+
# Returns an array of all known Resque keys in Redis. Redis' KEYS operation
|
178
|
+
# is O(N) for the keyspace, so be careful - this can be slow for big databases.
|
179
|
+
def keys
|
180
|
+
redis.keys("*").map do |key|
|
181
|
+
key.sub('resque:', '')
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Resque
|
2
|
+
# The Failure module provides an interface for working with different
|
3
|
+
# failure backends.
|
4
|
+
#
|
5
|
+
# You can use it to query the failure backend without knowing which specific
|
6
|
+
# backend is being used. For instance, the Resque web app uses it to display
|
7
|
+
# stats and other information.
|
8
|
+
module Failure
|
9
|
+
# Creates a new failure, which is delegated to the appropriate backend.
|
10
|
+
#
|
11
|
+
# Expects a hash with the following keys:
|
12
|
+
# :exception - The Exception object
|
13
|
+
# :worker - The Worker object who is reporting the failure
|
14
|
+
# :queue - The string name of the queue from which the job was pulled
|
15
|
+
# :payload - The job's payload
|
16
|
+
def self.create(options = {})
|
17
|
+
backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Sets the current backend. Expects a class descendent of
|
22
|
+
# `Resque::Failure::Base`.
|
23
|
+
#
|
24
|
+
# Example use:
|
25
|
+
# require 'resque/failure/hoptoad'
|
26
|
+
# Resque::Failure.backend = Resque::Failure::Hoptoad
|
27
|
+
def self.backend=(backend)
|
28
|
+
@backend = backend
|
29
|
+
end
|
30
|
+
|
31
|
+
# Returns the current backend class. If none has been set, falls
|
32
|
+
# back to `Resque::Failure::Redis`
|
33
|
+
def self.backend
|
34
|
+
return @backend if @backend
|
35
|
+
require 'resque/failure/redis'
|
36
|
+
@backend = Failure::Redis
|
37
|
+
end
|
38
|
+
|
39
|
+
# Returns the int count of how many failures we have seen.
|
40
|
+
def self.count
|
41
|
+
backend.count
|
42
|
+
end
|
43
|
+
|
44
|
+
# Returns an array of all the failures, paginated.
|
45
|
+
#
|
46
|
+
# `start` is the int of the first item in the page, `count` is the
|
47
|
+
# number of items to return.
|
48
|
+
def self.all(start = 0, count = 1)
|
49
|
+
backend.all(start, count)
|
50
|
+
end
|
51
|
+
|
52
|
+
# The string url of the backend's web interface, if any.
|
53
|
+
def self.url
|
54
|
+
backend.url
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# All Failure classes are expected to subclass Base.
|
4
|
+
#
|
5
|
+
# When a job fails, a new instance of your Failure backend is created
|
6
|
+
# and #save is called.
|
7
|
+
class Base
|
8
|
+
# The exception object raised by the failed job
|
9
|
+
attr_accessor :exception
|
10
|
+
|
11
|
+
# The worker object who detected the failure
|
12
|
+
attr_accessor :worker
|
13
|
+
|
14
|
+
# The string name of the queue from which the failed job was pulled
|
15
|
+
attr_accessor :queue
|
16
|
+
|
17
|
+
# The payload object associated with the failed job
|
18
|
+
attr_accessor :payload
|
19
|
+
|
20
|
+
def initialize(exception, worker, queue, payload)
|
21
|
+
@exception = exception
|
22
|
+
@worker = worker
|
23
|
+
@queue = queue
|
24
|
+
@payload = payload
|
25
|
+
end
|
26
|
+
|
27
|
+
# When a job fails, a new instance of your Failure backend is created
|
28
|
+
# and #save is called.
|
29
|
+
#
|
30
|
+
# This is where you POST or PUT or whatever to your Failure service.
|
31
|
+
def save
|
32
|
+
end
|
33
|
+
|
34
|
+
# The number of failures.
|
35
|
+
def self.count
|
36
|
+
0
|
37
|
+
end
|
38
|
+
|
39
|
+
# Returns a paginated array of failure objects.
|
40
|
+
def self.all(start = 0, count = 1)
|
41
|
+
[]
|
42
|
+
end
|
43
|
+
|
44
|
+
# A URL where someone can go to view failures.
|
45
|
+
def self.url
|
46
|
+
end
|
47
|
+
|
48
|
+
# Logging!
|
49
|
+
def log(message)
|
50
|
+
@worker.log(message)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Failure
|
5
|
+
# A Failure backend that sends exceptions raised by jobs to Hoptoad.
|
6
|
+
#
|
7
|
+
# To use it, put this code in an initializer, Rake task, or wherever:
|
8
|
+
#
|
9
|
+
# Resque::Failure::Hoptoad.configure do |config|
|
10
|
+
# config.api_key = 'blah'
|
11
|
+
# config.secure = true
|
12
|
+
# config.subdomain = 'your_hoptoad_subdomain'
|
13
|
+
# end
|
14
|
+
class Hoptoad < Base
|
15
|
+
class << self
|
16
|
+
attr_accessor :secure, :api_key, :subdomain
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.url
|
20
|
+
"http://#{subdomain}.hoptoadapp.com/" if subdomain
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.count
|
24
|
+
# We can't get the total # of errors from Hoptoad so we fake it
|
25
|
+
# by asking Resque how many errors it has seen.
|
26
|
+
Stat[:failed]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.configure
|
30
|
+
yield self
|
31
|
+
Resque::Failure.backend = self
|
32
|
+
end
|
33
|
+
|
34
|
+
def save
|
35
|
+
data = {
|
36
|
+
:api_key => api_key,
|
37
|
+
:error_class => exception.class.name,
|
38
|
+
:error_message => "#{exception.class.name}: #{exception.message}",
|
39
|
+
:backtrace => exception.backtrace,
|
40
|
+
:environment => {},
|
41
|
+
:session => {},
|
42
|
+
:request => {
|
43
|
+
:params => payload.merge(:worker => worker.to_s, :queue => queue.to_s)
|
44
|
+
}
|
45
|
+
}
|
46
|
+
|
47
|
+
send_to_hoptoad(:notice => data)
|
48
|
+
end
|
49
|
+
|
50
|
+
def send_to_hoptoad(data)
|
51
|
+
http = use_ssl? ? :https : :http
|
52
|
+
url = URI.parse("#{http}://hoptoadapp.com/notices/")
|
53
|
+
|
54
|
+
http = Net::HTTP.new(url.host, url.port)
|
55
|
+
headers = {
|
56
|
+
'Content-type' => 'application/json',
|
57
|
+
'Accept' => 'text/xml, application/xml'
|
58
|
+
}
|
59
|
+
|
60
|
+
http.read_timeout = 5 # seconds
|
61
|
+
http.open_timeout = 2 # seconds
|
62
|
+
http.use_ssl = use_ssl?
|
63
|
+
|
64
|
+
begin
|
65
|
+
response = http.post(url.path, Resque.encode(data), headers)
|
66
|
+
rescue TimeoutError => e
|
67
|
+
log "Timeout while contacting the Hoptoad server."
|
68
|
+
end
|
69
|
+
|
70
|
+
case response
|
71
|
+
when Net::HTTPSuccess then
|
72
|
+
log "Hoptoad Success: #{response.class}"
|
73
|
+
else
|
74
|
+
body = response.body if response.respond_to? :body
|
75
|
+
log "Hoptoad Failure: #{response.class}\n#{body}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def use_ssl?
|
80
|
+
self.class.secure
|
81
|
+
end
|
82
|
+
|
83
|
+
def api_key
|
84
|
+
self.class.api_key
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Resque
|
2
|
+
module Failure
|
3
|
+
# A Failure backend that stores exceptions in Redis. Very simple but
|
4
|
+
# works out of the box, along with support in the Resque web app.
|
5
|
+
class Redis < Base
|
6
|
+
def save
|
7
|
+
data = {
|
8
|
+
:failed_at => Time.now.to_s,
|
9
|
+
:payload => payload,
|
10
|
+
:error => exception.to_s,
|
11
|
+
:backtrace => exception.backtrace,
|
12
|
+
:worker => worker,
|
13
|
+
:queue => queue
|
14
|
+
}
|
15
|
+
data = Resque.encode(data)
|
16
|
+
Resque.redis.rpush(:failed, data)
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.count
|
20
|
+
Resque.redis.llen(:failed).to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.all(start = 0, count = 1)
|
24
|
+
Resque.list_range(:failed, start, count)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module Resque
|
2
|
+
# Methods used by various classes in Resque.
|
3
|
+
module Helpers
|
4
|
+
# Direct access to the Redis instance.
|
5
|
+
def redis
|
6
|
+
Resque.redis
|
7
|
+
end
|
8
|
+
|
9
|
+
# Given a Ruby object, returns a string suitable for storage in a
|
10
|
+
# queue.
|
11
|
+
def encode(object)
|
12
|
+
if defined? Yajl
|
13
|
+
Yajl::Encoder.encode(object)
|
14
|
+
else
|
15
|
+
JSON(object)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Given a string, returns a Ruby object.
|
20
|
+
def decode(object)
|
21
|
+
return unless object
|
22
|
+
|
23
|
+
if defined? Yajl
|
24
|
+
Yajl::Parser.parse(object)
|
25
|
+
else
|
26
|
+
JSON(object)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# Given a word with dashes, returns a camel cased version of it.
|
31
|
+
#
|
32
|
+
# classify('job-name') # => 'JobName'
|
33
|
+
def classify(dashed_word)
|
34
|
+
dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
|
35
|
+
end
|
36
|
+
|
37
|
+
# Given a camel cased word, returns the constant it represents
|
38
|
+
#
|
39
|
+
# constantize('JobName') # => JobName
|
40
|
+
def constantize(camel_cased_word)
|
41
|
+
camel_cased_word = camel_cased_word.to_s
|
42
|
+
|
43
|
+
if camel_cased_word.include?('-')
|
44
|
+
camel_cased_word = classify(camel_cased_word)
|
45
|
+
end
|
46
|
+
|
47
|
+
names = camel_cased_word.split('::')
|
48
|
+
names.shift if names.empty? || names.first.empty?
|
49
|
+
|
50
|
+
constant = Object
|
51
|
+
names.each do |name|
|
52
|
+
constant = constant.const_get(name) || constant.const_missing(name)
|
53
|
+
end
|
54
|
+
constant
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|