job_queue 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +58 -0
- data/VERSION.yml +4 -0
- data/lib/job_queue/adapters/amqp_adapter.rb +28 -0
- data/lib/job_queue/adapters/beanstalk_adapter.rb +106 -0
- data/lib/job_queue/adapters/test_adapter.rb +53 -0
- data/lib/job_queue/adapters/verbose_adapter.rb +23 -0
- data/lib/job_queue/job_queue.rb +71 -0
- data/lib/job_queue.rb +7 -0
- data/spec/amqp_adapter_spec.rb +8 -0
- data/spec/beanstalk_adapter_spec.rb +322 -0
- data/spec/common_adapter_spec.rb +62 -0
- data/spec/job_queue_spec.rb +5 -0
- data/spec/spec_helper.rb +13 -0
- data/spec/test_adapter_spec.rb +20 -0
- data/spec/verbose_adapter_spec.rb +15 -0
- metadata +70 -0
data/README.markdown
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
JobQueue
|
2
|
+
========
|
3
|
+
|
4
|
+
`job_queue` allows you to use lots of message queues with exactly the same interface so you don't need to worry about which queue to pick :)
|
5
|
+
|
6
|
+
This should get you started:
|
7
|
+
|
8
|
+
require 'rubygems'
|
9
|
+
require 'job_queue'
|
10
|
+
|
11
|
+
Before you can do anything you must specify an adapter to use
|
12
|
+
|
13
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
14
|
+
|
15
|
+
Jobs can then be simply added to the queue
|
16
|
+
|
17
|
+
JobQueue.put("flubble bubble")
|
18
|
+
|
19
|
+
In your workers you'll want to subscribe to a queue
|
20
|
+
|
21
|
+
JobQueue.subscribe do |job|
|
22
|
+
puts job
|
23
|
+
end
|
24
|
+
|
25
|
+
This subscribe block takes care of waiting for the next job to arrive and the block is passed exactly what you passed in. If you want to exit the loop just throw :stop.
|
26
|
+
|
27
|
+
JobQueue.subscribe do |job|
|
28
|
+
# Wait - I changed my mind!
|
29
|
+
throw :stop
|
30
|
+
end
|
31
|
+
|
32
|
+
What should you put on the queue
|
33
|
+
--------------------------------
|
34
|
+
|
35
|
+
You might love Ruby right now, but why lock yourself in? Often the kinds of things you use queues for are the kind of things you'll want to optimize. This is a good place to start:
|
36
|
+
|
37
|
+
JSON.generate({:some => "hash"})
|
38
|
+
JSON.parse(job)
|
39
|
+
|
40
|
+
Can you show me a nice processing daemon?
|
41
|
+
-----------------------------------------
|
42
|
+
|
43
|
+
Yes. Just a minute...
|
44
|
+
|
45
|
+
Adapters
|
46
|
+
========
|
47
|
+
|
48
|
+
Take your pick! Right now we have:
|
49
|
+
|
50
|
+
Beanstalk
|
51
|
+
---------
|
52
|
+
<http://xph.us/software/beanstalkd/>
|
53
|
+
|
54
|
+
AMQP
|
55
|
+
----
|
56
|
+
<http://github.com/tmm1/amqp/>
|
57
|
+
|
58
|
+
You need to run all your code within an eventmachine loop to use AMQP.
|
data/VERSION.yml
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'mq'
|
2
|
+
|
3
|
+
class JobQueue::AMQPAdapter
|
4
|
+
def initialize(options = {})
|
5
|
+
amq = MQ.new
|
6
|
+
@exchange = amq.direct('photo', :durable => true)
|
7
|
+
@queue = amq.queue('photo_worker', :durable => true)
|
8
|
+
@queue.bind(@exchange)
|
9
|
+
end
|
10
|
+
|
11
|
+
def put(string)
|
12
|
+
@queue.publish(string, :persistent => true)
|
13
|
+
end
|
14
|
+
|
15
|
+
def subscribe(error_report, &block)
|
16
|
+
EM.add_periodic_timer(0) do
|
17
|
+
begin
|
18
|
+
@queue.pop do |header, body|
|
19
|
+
next unless body
|
20
|
+
JobQueue.logger.debug "AMQP received #{body}"
|
21
|
+
yield body
|
22
|
+
end
|
23
|
+
rescue => e
|
24
|
+
error_report.call(job.body, e)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'beanstalk-client'
|
2
|
+
require 'timeout'
|
3
|
+
|
4
|
+
class JobQueue::BeanstalkAdapter
|
5
|
+
def initialize(options = {})
|
6
|
+
@hosts = options[:hosts] || 'localhost:11300'
|
7
|
+
end
|
8
|
+
|
9
|
+
def put(string, queue, priority, ttr)
|
10
|
+
ttr = ttr.floor #rounding because Beanstalk doesnt accept float numbers
|
11
|
+
raise JobQueue::ArgumentError, "TTR must be greater than 1" if ttr < 2
|
12
|
+
|
13
|
+
delay = 0
|
14
|
+
job_info = beanstalk_pool(queue).put_and_report_conn \
|
15
|
+
string, priority, delay, ttr
|
16
|
+
"#{job_info[:host]}_#{job_info[:id]}"
|
17
|
+
rescue Beanstalk::NotConnected
|
18
|
+
raise JobQueue::NoConnectionAvailable
|
19
|
+
end
|
20
|
+
|
21
|
+
def subscribe(error_report, cleanup_task, queue, &block)
|
22
|
+
pool = BeanstalkPoolFix.new([@hosts].flatten, queue)
|
23
|
+
loop do
|
24
|
+
begin
|
25
|
+
job = pool.reserve(1)
|
26
|
+
time_left = job.stats["time-left"]
|
27
|
+
JobQueue.logger.debug "Beanstalk received #{job.body}"
|
28
|
+
Timeout::timeout([time_left - 1, 1].max) do
|
29
|
+
yield job.body
|
30
|
+
end
|
31
|
+
job.delete
|
32
|
+
rescue Timeout::Error
|
33
|
+
cleanup_task.call(job.body)
|
34
|
+
JobQueue.logger.warn "Job timed out"
|
35
|
+
begin
|
36
|
+
job.delete
|
37
|
+
rescue Beanstalk::NotFoundError
|
38
|
+
JobQueue.logger.error "Job timed out and could not be deleted"
|
39
|
+
end
|
40
|
+
rescue Beanstalk::TimedOut
|
41
|
+
# Do nothing - retry to reseve (from another host?)
|
42
|
+
rescue => e
|
43
|
+
if job
|
44
|
+
error_report.call(job.body, e)
|
45
|
+
begin
|
46
|
+
job.delete
|
47
|
+
rescue Beanstalk::NotFoundError
|
48
|
+
JobQueue.logger.error "Job failed but could not be deleted"
|
49
|
+
end
|
50
|
+
else
|
51
|
+
JobQueue.logger.error "Unhandled exception: #{e.message}\n" \
|
52
|
+
"#{e.backtrace.join("\n")}\n"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def job_stats(job_id)
|
59
|
+
host, id = job_id.split('_')
|
60
|
+
beanstalk_pool.job_stats(id).select { |k, v| k == host }[0][1]
|
61
|
+
rescue Beanstalk::NotFoundError
|
62
|
+
nil
|
63
|
+
end
|
64
|
+
|
65
|
+
def queue_length(queue)
|
66
|
+
beanstalk_pool.stats_tube(queue)["total-jobs"]
|
67
|
+
rescue Beanstalk::NotFoundError
|
68
|
+
0
|
69
|
+
end
|
70
|
+
|
71
|
+
def beanstalk_pool(queue='default')
|
72
|
+
@beanstalk_pools ||= {}
|
73
|
+
@beanstalk_pools[queue] ||= begin
|
74
|
+
BeanstalkPoolFix.new([@hosts].flatten, queue)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
class BeanstalkPoolFix < Beanstalk::Pool
|
79
|
+
def put_and_report_conn(body, pri=65536, delay=0, ttr=120)
|
80
|
+
send_to_rand_conn_and_report(:put, body, pri, delay, ttr)
|
81
|
+
end
|
82
|
+
|
83
|
+
def send_to_rand_conn_and_report(*args)
|
84
|
+
connect()
|
85
|
+
retry_wrap{
|
86
|
+
conn = pick_connection
|
87
|
+
{:host => conn.addr, :id => call_wrap(conn, *args)}
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
def job_stats(id)
|
92
|
+
make_hash(send_to_all_conns(:job_stats, id))
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
def call_wrap(c, *args)
|
98
|
+
self.last_conn = c
|
99
|
+
c.send(*args)
|
100
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Beanstalk::UnexpectedResponse => ex
|
101
|
+
# puts "Beanstalk exception: #{ex.class}" # Useful for debugging
|
102
|
+
self.remove(c) unless ex.class == Beanstalk::TimedOut
|
103
|
+
raise ex
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# This adapter is designed for testing purposes.
|
2
|
+
#
|
3
|
+
# Features supported:
|
4
|
+
#
|
5
|
+
# named queues: yes
|
6
|
+
# priority: no
|
7
|
+
# ttr: no
|
8
|
+
#
|
9
|
+
# Additionally this queue can be inspeced with JobQueue.adapter.queue('name')
|
10
|
+
#
|
11
|
+
class JobQueue::TestAdapter
|
12
|
+
def initialize(options = {})
|
13
|
+
@queues = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def put(string, queue, priority, ttr)
|
17
|
+
@queues[queue] ||= []
|
18
|
+
@queues[queue] << string
|
19
|
+
end
|
20
|
+
|
21
|
+
def subscribe(error_report, cleanup_task, queue, &block)
|
22
|
+
loop do
|
23
|
+
begin
|
24
|
+
if get_queue(queue).empty?
|
25
|
+
sleep 0.1
|
26
|
+
else
|
27
|
+
job = get_queue(queue).shift
|
28
|
+
yield job
|
29
|
+
end
|
30
|
+
rescue => e
|
31
|
+
error_report.call(job, e)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Additional method for TestAdapter to allow easy queue inspection with
|
37
|
+
#
|
38
|
+
# JobQueue.adapter.queue('foo')
|
39
|
+
#
|
40
|
+
def queue(queue = 'default')
|
41
|
+
get_queue(queue)
|
42
|
+
end
|
43
|
+
|
44
|
+
def queue_length(queue)
|
45
|
+
@queues[queue].size
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def get_queue(queue)
|
51
|
+
@queues[queue] || []
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# This isn't a queue at all, it just writes to standard output.
|
2
|
+
#
|
3
|
+
# It might be useful for testing.
|
4
|
+
#
|
5
|
+
class JobQueue::VerboseAdapter
|
6
|
+
def initialize(options = {})
|
7
|
+
|
8
|
+
end
|
9
|
+
|
10
|
+
def put(string, queue, priority, ttr)
|
11
|
+
JobQueue.logger.debug "===== NEW JOB ADDED TO QUEUE ===="
|
12
|
+
JobQueue.logger.debug string
|
13
|
+
JobQueue.logger.debug "===== END OF MESSAGE ============"
|
14
|
+
end
|
15
|
+
|
16
|
+
def subscribe(error_report, &block)
|
17
|
+
raise "Not implemented. Use a better adapter!!"
|
18
|
+
end
|
19
|
+
|
20
|
+
def queue_length(queue)
|
21
|
+
raise "Not supported"
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# JobQueue abstracts the task of adding work to a queue.
|
2
|
+
#
|
3
|
+
# Beanstalk is fantastic, but maybe not "enterprise grade".
|
4
|
+
#
|
5
|
+
# AMQP is fantastic, but it's bloody complex and has to run inside an
|
6
|
+
# eventmachine loop.
|
7
|
+
#
|
8
|
+
# Take your pick!
|
9
|
+
#
|
10
|
+
# Before use, an adapter must be chosen:
|
11
|
+
#
|
12
|
+
# JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
13
|
+
#
|
14
|
+
# Jobs can then be simply added to the queue with
|
15
|
+
#
|
16
|
+
# JobQueue.put("flubble bubble")
|
17
|
+
#
|
18
|
+
class JobQueue
|
19
|
+
class << self
|
20
|
+
attr_accessor :adapter
|
21
|
+
attr_accessor :logger
|
22
|
+
|
23
|
+
def logger
|
24
|
+
@logger ||= begin
|
25
|
+
logger = Logger.new(STDOUT)
|
26
|
+
logger.level = Logger::WARN
|
27
|
+
logger.debug("Created logger")
|
28
|
+
logger
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.put(string, options = {})
|
34
|
+
queue = options[:queue] || 'default'
|
35
|
+
priority = options[:priority] || 50
|
36
|
+
ttr = options[:ttr] || 60
|
37
|
+
adapter.put(string, queue, priority, ttr)
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.subscribe(options = {}, &block)
|
41
|
+
queue = options[:queue] || 'default'
|
42
|
+
error_report = options[:error_report] || Proc.new do |job_body, e|
|
43
|
+
JobQueue.logger.error \
|
44
|
+
"Job failed\n" \
|
45
|
+
"==========\n" \
|
46
|
+
"Job content: #{job_body.inspect}\n" \
|
47
|
+
"Exception: #{e.message}\n" \
|
48
|
+
"#{e.backtrace.join("\n")}\n" \
|
49
|
+
"\n"
|
50
|
+
end
|
51
|
+
cleanup_task = options[:cleanup] || lambda {}
|
52
|
+
catch :stop do
|
53
|
+
adapter.subscribe(error_report, cleanup_task, queue, &block)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns a hash of info (exact details dependent on adapter)
|
58
|
+
def self.job_stats(job_id)
|
59
|
+
adapter.job_stats(job_id)
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.queue_length(queue = nil)
|
63
|
+
adapter.queue_length(queue)
|
64
|
+
end
|
65
|
+
|
66
|
+
class NoConnectionAvailable < RuntimeError
|
67
|
+
end
|
68
|
+
|
69
|
+
class ArgumentError < ::ArgumentError
|
70
|
+
end
|
71
|
+
end
|
data/lib/job_queue.rb
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'job_queue/job_queue'
|
3
|
+
|
4
|
+
JobQueue.autoload 'AMQPAdapter', 'job_queue/adapters/amqp_adapter'
|
5
|
+
JobQueue.autoload 'BeanstalkAdapter', 'job_queue/adapters/beanstalk_adapter'
|
6
|
+
JobQueue.autoload 'TestAdapter', 'job_queue/adapters/test_adapter'
|
7
|
+
JobQueue.autoload 'VerboseAdapter', 'job_queue/adapters/verbose_adapter'
|
@@ -0,0 +1,322 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
require File.dirname(__FILE__) + '/common_adapter_spec'
|
3
|
+
|
4
|
+
describe JobQueue::BeanstalkAdapter do
|
5
|
+
before :each do
|
6
|
+
# On OSX we the -d flag doesn't work for beanstalk 1.3. This is a
|
7
|
+
# workaround for that issue. We sleep a little to let processes start.
|
8
|
+
system "beanstalkd -p 10001 &"
|
9
|
+
system "beanstalkd -p 10002 &"
|
10
|
+
system "beanstalkd -p 11300 &"
|
11
|
+
sleep 0.1
|
12
|
+
end
|
13
|
+
|
14
|
+
after :each do
|
15
|
+
system "killall beanstalkd"
|
16
|
+
end
|
17
|
+
|
18
|
+
describe '#new' do
|
19
|
+
before(:each) do
|
20
|
+
@pool = JobQueue::BeanstalkAdapter::BeanstalkPoolFix.new([
|
21
|
+
'localhost:11300'
|
22
|
+
])
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should default to localhost:11300" do
|
26
|
+
JobQueue::BeanstalkAdapter::BeanstalkPoolFix.should_receive(:new).with(
|
27
|
+
['localhost:11300'],
|
28
|
+
"default"
|
29
|
+
).and_return @pool
|
30
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
31
|
+
JobQueue.put('test')
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should accept one beanstalk instance" do
|
35
|
+
JobQueue::BeanstalkAdapter::BeanstalkPoolFix.should_receive(:new).with(
|
36
|
+
['12.34.56.78:12345'],
|
37
|
+
'default'
|
38
|
+
).and_return(@pool)
|
39
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new(
|
40
|
+
:hosts => '12.34.56.78:12345'
|
41
|
+
)
|
42
|
+
JobQueue.put('test')
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should allow multiple beanstalk instances" do
|
46
|
+
JobQueue::BeanstalkAdapter::BeanstalkPoolFix.should_receive(:new).with(
|
47
|
+
['12.34.56.78:12345', '87.65.43.21:54321'],
|
48
|
+
'default'
|
49
|
+
).and_return(@pool)
|
50
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
|
51
|
+
:hosts => ['12.34.56.78:12345', '87.65.43.21:54321']
|
52
|
+
})
|
53
|
+
JobQueue.put('test')
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
describe "put" do
|
58
|
+
before :each do
|
59
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
60
|
+
end
|
61
|
+
|
62
|
+
it "should return the job id" do
|
63
|
+
job_id = JobQueue.put("hello 1")
|
64
|
+
job_id.should == "localhost:11300_1"
|
65
|
+
end
|
66
|
+
|
67
|
+
it "should assign job priority" do
|
68
|
+
jobs = ["1","2","3"]
|
69
|
+
JobQueue.put(jobs[2], :priority => 3)
|
70
|
+
JobQueue.put(jobs[1], :priority => 2)
|
71
|
+
JobQueue.put(jobs[0], :priority => 1)
|
72
|
+
|
73
|
+
jobs_received = []
|
74
|
+
should_not_timeout(0.5) {
|
75
|
+
index = 0
|
76
|
+
JobQueue.subscribe do |job_body|
|
77
|
+
index += 1
|
78
|
+
jobs_received << job_body
|
79
|
+
throw :stop if index == 3
|
80
|
+
end
|
81
|
+
}
|
82
|
+
|
83
|
+
jobs_received.should == jobs
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should be able to retrieve job stats by id" do
|
87
|
+
job_id = JobQueue.put("hello 1")
|
88
|
+
job_id.should == "localhost:11300_1"
|
89
|
+
JobQueue.put("hello 2")
|
90
|
+
stats = JobQueue.job_stats("localhost:11300_1")
|
91
|
+
|
92
|
+
stats["id"].should == 1
|
93
|
+
stats["tube"].should == "default"
|
94
|
+
end
|
95
|
+
|
96
|
+
it "should raise error when no connections exist" do
|
97
|
+
system "killall beanstalkd"
|
98
|
+
lambda {
|
99
|
+
JobQueue.put('test')
|
100
|
+
}.should raise_error(JobQueue::NoConnectionAvailable)
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should succeed when one connection fails" do
|
104
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
|
105
|
+
:hosts => ['localhost:10001', 'localhost:666']
|
106
|
+
})
|
107
|
+
10.times{ job_id = JobQueue.put("hello 1")}
|
108
|
+
end
|
109
|
+
|
110
|
+
it "should raise an error if a ttr of < 2 is specified" do
|
111
|
+
lambda {
|
112
|
+
JobQueue.put('test', :ttr => 1.9)
|
113
|
+
}.should raise_error(JobQueue::ArgumentError)
|
114
|
+
|
115
|
+
lambda {
|
116
|
+
JobQueue.put('test', :ttr => 2)
|
117
|
+
}.should_not raise_error(JobQueue::ArgumentError)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
describe "subscribe" do
|
122
|
+
before :each do
|
123
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should delete a job once it has been succesfully excecuted" do
|
127
|
+
job_id = JobQueue.put('testdeleted')
|
128
|
+
JobQueue.put('foo')
|
129
|
+
index = 0
|
130
|
+
JobQueue.subscribe do |body|
|
131
|
+
index += 1
|
132
|
+
throw :stop if index == 2
|
133
|
+
end
|
134
|
+
JobQueue.job_stats(job_id).should be_nil
|
135
|
+
end
|
136
|
+
|
137
|
+
it "should report and error and delete the job if a job times out" do
|
138
|
+
job_id = JobQueue.put("job1", :ttr => 2)
|
139
|
+
JobQueue.put('test')
|
140
|
+
|
141
|
+
JobQueue.logger.should_receive(:warn).with("Job timed out")
|
142
|
+
|
143
|
+
index = 0
|
144
|
+
JobQueue.subscribe do |body|
|
145
|
+
index += 1
|
146
|
+
throw :stop if index == 2
|
147
|
+
sleep 2.2
|
148
|
+
end
|
149
|
+
|
150
|
+
JobQueue.job_stats(job_id).should be_nil
|
151
|
+
end
|
152
|
+
|
153
|
+
it "should allow a client to cleanup if a job times out" do
|
154
|
+
JobQueue.put('jobcleanup', :ttr => 2)
|
155
|
+
JobQueue.put('test')
|
156
|
+
|
157
|
+
cleanup = nil
|
158
|
+
|
159
|
+
index = 0
|
160
|
+
JobQueue.subscribe(:cleanup => lambda { |job| FileUtils.rm(job) }) do |body|
|
161
|
+
file = File.open(body, 'w')
|
162
|
+
file << "hello"
|
163
|
+
file.flush
|
164
|
+
|
165
|
+
index += 1
|
166
|
+
throw :stop if index == 2
|
167
|
+
sleep 2.2
|
168
|
+
end
|
169
|
+
|
170
|
+
File.exists?('jobcleanup').should be_false
|
171
|
+
end
|
172
|
+
|
173
|
+
# This test is for a patch that fixes a connection leaking issue in
|
174
|
+
# beanstalk-client 1.0.2
|
175
|
+
it "should not open more connections to beanstalk over time" do
|
176
|
+
# Every 1.5 seconds, add a new job to the queue and check how many
|
177
|
+
# connections are currently open according to beanstalkd.
|
178
|
+
connections = []
|
179
|
+
Thread.new do
|
180
|
+
pool = Beanstalk::Pool.new(["localhost:11300"])
|
181
|
+
loop do
|
182
|
+
sleep 1.5
|
183
|
+
JobQueue.put("job")
|
184
|
+
connections << pool.stats["total-connections"]
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
# Subscribe for 3 loops - gives time for a few timeouts to occur (1s)
|
189
|
+
i = 0
|
190
|
+
JobQueue.subscribe do |job|
|
191
|
+
i += 1
|
192
|
+
throw :stop if i == 3
|
193
|
+
end
|
194
|
+
|
195
|
+
# The number of connections should have been constant
|
196
|
+
connections.uniq.size.should == 1
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
describe "job_stats" do
|
201
|
+
before :each do
|
202
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
203
|
+
end
|
204
|
+
|
205
|
+
it "should gracefully deal with jobs where connection no longer exists" do
|
206
|
+
JobQueue.job_stats("localhost:11305_1").should be_nil
|
207
|
+
end
|
208
|
+
|
209
|
+
it "should gracefully deal with jobs where job doesn't exist" do
|
210
|
+
JobQueue.job_stats("localhost:11300_1").should be_nil
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
describe "when connecting to one instance" do
|
215
|
+
before :each do
|
216
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
217
|
+
end
|
218
|
+
|
219
|
+
describe "common" do
|
220
|
+
it_should_behave_like "JobQueue adapter named queues"
|
221
|
+
it_should_behave_like "JobQueue adapter queue length"
|
222
|
+
end
|
223
|
+
|
224
|
+
it "should write onto queue and fetch stuff back off" do
|
225
|
+
JobQueue.put("hello")
|
226
|
+
|
227
|
+
should_not_timeout {
|
228
|
+
JobQueue.subscribe do |job|
|
229
|
+
@job = job
|
230
|
+
throw :stop
|
231
|
+
end
|
232
|
+
}
|
233
|
+
|
234
|
+
@job.should == "hello"
|
235
|
+
end
|
236
|
+
|
237
|
+
it "should output message if error raised in job" do
|
238
|
+
JobQueue.put("hello")
|
239
|
+
JobQueue.put("hello2")
|
240
|
+
|
241
|
+
JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
|
242
|
+
|
243
|
+
should_not_timeout {
|
244
|
+
index = 0
|
245
|
+
JobQueue.subscribe do |job|
|
246
|
+
index +=1
|
247
|
+
raise 'foo' if index == 1
|
248
|
+
throw :stop
|
249
|
+
end
|
250
|
+
}
|
251
|
+
end
|
252
|
+
|
253
|
+
it "should use error_report block if supplied" do
|
254
|
+
JobQueue.put("hello")
|
255
|
+
JobQueue.put("hello2")
|
256
|
+
|
257
|
+
error_report = Proc.new do |job, e|
|
258
|
+
JobQueue.logger.error "Yikes that broke matey!"
|
259
|
+
end
|
260
|
+
|
261
|
+
JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
|
262
|
+
|
263
|
+
should_not_timeout {
|
264
|
+
index = 0
|
265
|
+
JobQueue.subscribe(:error_report => error_report) do |job|
|
266
|
+
index +=1
|
267
|
+
raise 'foo' if index == 1
|
268
|
+
throw :stop
|
269
|
+
end
|
270
|
+
}
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
describe "when connecting to multiple instances" do
|
275
|
+
before :each do
|
276
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
|
277
|
+
:hosts => ['localhost:10001', 'localhost:10002']
|
278
|
+
})
|
279
|
+
end
|
280
|
+
|
281
|
+
describe "common" do
|
282
|
+
it_should_behave_like "JobQueue adapter queue length"
|
283
|
+
end
|
284
|
+
|
285
|
+
it "should be possible to put jobs" do
|
286
|
+
JobQueue.put('test')
|
287
|
+
JobQueue.subscribe do |job|
|
288
|
+
job.should == 'test'
|
289
|
+
throw :stop
|
290
|
+
end
|
291
|
+
end
|
292
|
+
|
293
|
+
# TODO: This test is brittle.
|
294
|
+
it "should be possible to retrieve all jobs supplied" do
|
295
|
+
# Put some jobs on the queue
|
296
|
+
jobs = []
|
297
|
+
(1..8).each do |i|
|
298
|
+
body = i
|
299
|
+
JobQueue.put("#{body}")
|
300
|
+
jobs << body
|
301
|
+
end
|
302
|
+
|
303
|
+
should_not_timeout(3.5) {
|
304
|
+
JobQueue.subscribe do |job|
|
305
|
+
jobs.delete job.to_i
|
306
|
+
throw :stop if jobs.empty?
|
307
|
+
end
|
308
|
+
}
|
309
|
+
end
|
310
|
+
|
311
|
+
it "should not log any errors when reserve times out" do
|
312
|
+
JobQueue.logger.should_not_receive(:error)
|
313
|
+
begin
|
314
|
+
Timeout::timeout(1.5) do
|
315
|
+
JobQueue.subscribe { |job| }
|
316
|
+
end
|
317
|
+
rescue Timeout::Error
|
318
|
+
#Do nothing - timeout expected
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
shared_examples_for 'JobQueue adapter basic' do
|
4
|
+
it "should write onto queue and fetch stuff back off" do
|
5
|
+
JobQueue.put("hello")
|
6
|
+
|
7
|
+
JobQueue.subscribe do |job|
|
8
|
+
@job = job
|
9
|
+
throw :stop
|
10
|
+
end
|
11
|
+
|
12
|
+
@job.should == "hello"
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should pull items off in the order the were added" do
|
16
|
+
JobQueue.put("foo")
|
17
|
+
JobQueue.put("bar")
|
18
|
+
|
19
|
+
retrieved_jobs = []
|
20
|
+
|
21
|
+
begin
|
22
|
+
Timeout::timeout(0.5) do
|
23
|
+
JobQueue.subscribe do |job|
|
24
|
+
retrieved_jobs << job
|
25
|
+
end
|
26
|
+
end
|
27
|
+
rescue Timeout::Error
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
retrieved_jobs[0].should == "foo"
|
32
|
+
retrieved_jobs[1].should == "bar"
|
33
|
+
retrieved_jobs[2].should == nil
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
shared_examples_for "JobQueue adapter named queues" do
|
38
|
+
it "should put jobs onto a named queue and only read off that queue" do
|
39
|
+
JobQueue.put("hello", :queue => "test")
|
40
|
+
lambda {
|
41
|
+
Timeout.timeout(0.1) do
|
42
|
+
JobQueue.subscribe(:queue => "foo") do |job|
|
43
|
+
throw :stop
|
44
|
+
end
|
45
|
+
end
|
46
|
+
}.should raise_error(Timeout::Error)
|
47
|
+
should_not_timeout {
|
48
|
+
JobQueue.subscribe(:queue => "test") do |body|
|
49
|
+
body.should == 'hello'
|
50
|
+
throw :stop
|
51
|
+
end
|
52
|
+
}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
shared_examples_for "JobQueue adapter queue length" do
|
57
|
+
it "should report the length of the named queue" do
|
58
|
+
JobQueue.queue_length('test').should == 0
|
59
|
+
5.times { JobQueue.put("hello", :queue => "test") }
|
60
|
+
JobQueue.queue_length('test').should == 5
|
61
|
+
end
|
62
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
$TESTING=true
|
2
|
+
$:.push File.join(File.dirname(__FILE__), '..', 'lib')
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
require 'job_queue'
|
6
|
+
|
7
|
+
def should_not_timeout(timeout = 0.1)
|
8
|
+
lambda {
|
9
|
+
Timeout.timeout(timeout) do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
}.should_not raise_error(Timeout::Error)
|
13
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
require File.dirname(__FILE__) + '/common_adapter_spec'
|
3
|
+
|
4
|
+
describe JobQueue::TestAdapter do
|
5
|
+
before :all do
|
6
|
+
JobQueue.adapter = JobQueue::TestAdapter.new
|
7
|
+
end
|
8
|
+
|
9
|
+
it_should_behave_like 'JobQueue adapter basic'
|
10
|
+
|
11
|
+
it_should_behave_like "JobQueue adapter named queues"
|
12
|
+
|
13
|
+
it_should_behave_like "JobQueue adapter queue length"
|
14
|
+
|
15
|
+
it "should allow queue inspection as a hash" do
|
16
|
+
JobQueue.adapter.queue.should == []
|
17
|
+
JobQueue.put('hello')
|
18
|
+
JobQueue.adapter.queue.should == ['hello']
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
describe JobQueue::VerboseAdapter do
|
4
|
+
before :all do
|
5
|
+
JobQueue.adapter = JobQueue::VerboseAdapter.new
|
6
|
+
end
|
7
|
+
|
8
|
+
it "should write onto queue and output a very verbose message to stdout" do
|
9
|
+
JobQueue.logger.should_receive(:debug).with("===== NEW JOB ADDED TO QUEUE ====")
|
10
|
+
JobQueue.logger.should_receive(:debug).with("hello")
|
11
|
+
JobQueue.logger.should_receive(:debug).with("===== END OF MESSAGE ============")
|
12
|
+
|
13
|
+
JobQueue.put("hello")
|
14
|
+
end
|
15
|
+
end
|
metadata
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: job_queue
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.9
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Martyn Loughran
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-08-26 00:00:00 +01:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: JobQueue means you don't have to worry about your queue any more!
|
17
|
+
email: me@mloughran.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- README.markdown
|
26
|
+
- VERSION.yml
|
27
|
+
- lib/job_queue/adapters/amqp_adapter.rb
|
28
|
+
- lib/job_queue/adapters/beanstalk_adapter.rb
|
29
|
+
- lib/job_queue/adapters/test_adapter.rb
|
30
|
+
- lib/job_queue/adapters/verbose_adapter.rb
|
31
|
+
- lib/job_queue/job_queue.rb
|
32
|
+
- lib/job_queue.rb
|
33
|
+
- spec/amqp_adapter_spec.rb
|
34
|
+
- spec/beanstalk_adapter_spec.rb
|
35
|
+
- spec/common_adapter_spec.rb
|
36
|
+
- spec/job_queue_spec.rb
|
37
|
+
- spec/spec_helper.rb
|
38
|
+
- spec/test_adapter_spec.rb
|
39
|
+
- spec/verbose_adapter_spec.rb
|
40
|
+
has_rdoc: true
|
41
|
+
homepage: http://github.com/mloughran/job_queue
|
42
|
+
licenses: []
|
43
|
+
|
44
|
+
post_install_message:
|
45
|
+
rdoc_options:
|
46
|
+
- --inline-source
|
47
|
+
- --charset=UTF-8
|
48
|
+
require_paths:
|
49
|
+
- lib
|
50
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: "0"
|
55
|
+
version:
|
56
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: "0"
|
61
|
+
version:
|
62
|
+
requirements: []
|
63
|
+
|
64
|
+
rubyforge_project:
|
65
|
+
rubygems_version: 1.3.5
|
66
|
+
signing_key:
|
67
|
+
specification_version: 3
|
68
|
+
summary: JobQueue means you don't have to worry about your queue any more!
|
69
|
+
test_files: []
|
70
|
+
|