mloughran-job_queue 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION.yml +2 -2
- data/lib/job_queue/adapters/beanstalk_adapter.rb +21 -11
- data/lib/job_queue/job_queue.rb +12 -8
- data/spec/beanstalk_adapter_spec.rb +147 -40
- metadata +10 -15
- data/LICENSE +0 -20
- data/Rakefile +0 -38
data/VERSION.yml
CHANGED
@@ -2,25 +2,35 @@ require 'beanstalk-client'
|
|
2
2
|
|
3
3
|
class JobQueue::BeanstalkAdapter
|
4
4
|
def initialize(options = {})
|
5
|
-
|
6
|
-
port = options[:port] || 11300
|
7
|
-
@beanstalk = Beanstalk::Pool.new(["#{host}:#{port}"])
|
5
|
+
@hosts = options[:hosts] || 'localhost:11300'
|
8
6
|
end
|
9
7
|
|
10
|
-
def put(string)
|
11
|
-
|
8
|
+
def put(string, queue, priority)
|
9
|
+
beanstalk_pool(queue).put(string)
|
12
10
|
end
|
13
11
|
|
14
|
-
def subscribe(error_report, &block)
|
12
|
+
def subscribe(error_report, queue, &block)
|
13
|
+
pool = Beanstalk::Pool.new([@hosts].flatten, queue)
|
15
14
|
loop do
|
16
15
|
begin
|
17
|
-
job =
|
16
|
+
job = pool.reserve(1)
|
18
17
|
JobQueue.logger.info "Beanstalk received #{job.body}"
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
18
|
+
begin
|
19
|
+
yield job.body
|
20
|
+
rescue => e
|
21
|
+
error_report.call(job.body, e)
|
22
|
+
job.delete
|
23
|
+
end
|
24
|
+
rescue Beanstalk::TimedOut
|
25
|
+
# Do nothing - retry to reseve (from another host?)
|
23
26
|
end
|
24
27
|
end
|
25
28
|
end
|
29
|
+
|
30
|
+
def beanstalk_pool(queue)
|
31
|
+
@beanstalk_pools ||= {}
|
32
|
+
@beanstalk_pools[queue] ||= begin
|
33
|
+
Beanstalk::Pool.new([@hosts].flatten, queue)
|
34
|
+
end
|
35
|
+
end
|
26
36
|
end
|
data/lib/job_queue/job_queue.rb
CHANGED
@@ -30,23 +30,27 @@ class JobQueue
|
|
30
30
|
end
|
31
31
|
end
|
32
32
|
|
33
|
-
def self.put(string)
|
34
|
-
|
33
|
+
def self.put(string, options = {})
|
34
|
+
queue = options[:queue] || 'default'
|
35
|
+
priority = options[:priority] || 50
|
36
|
+
adapter.put(string, queue, priority)
|
35
37
|
end
|
36
38
|
|
37
|
-
def self.subscribe(
|
38
|
-
|
39
|
-
|
39
|
+
def self.subscribe(options = {}, &block)
|
40
|
+
queue = options[:queue] || 'default'
|
41
|
+
error_report = options[:error_report] || begin
|
42
|
+
Proc.new do |job_body, e|
|
40
43
|
JobQueue.logger.error \
|
41
44
|
"Job failed\n" \
|
42
45
|
"==========\n" \
|
43
|
-
"Job content: #{
|
46
|
+
"Job content: #{job_body.inspect}\n" \
|
44
47
|
"Exception: #{e.message}\n" \
|
45
48
|
"#{e.backtrace.join("\n")}\n" \
|
46
49
|
"\n"
|
47
50
|
end
|
48
|
-
|
49
|
-
|
51
|
+
end
|
52
|
+
catch :stop do
|
53
|
+
adapter.subscribe(error_report, queue, &block)
|
50
54
|
end
|
51
55
|
end
|
52
56
|
end
|
@@ -1,60 +1,167 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
3
|
describe JobQueue::BeanstalkAdapter do
|
4
|
-
before :
|
5
|
-
|
4
|
+
before :each do
|
5
|
+
system "beanstalkd -p 10001 -d"
|
6
|
+
system "beanstalkd -p 10002 -d"
|
7
|
+
system "beanstalkd -p 11300 -d"
|
8
|
+
end
|
9
|
+
|
10
|
+
after :each do
|
11
|
+
system "killall beanstalkd"
|
6
12
|
end
|
7
13
|
|
8
14
|
describe '#new' do
|
15
|
+
before(:each) do
|
16
|
+
@pool = Beanstalk::Pool.new(['localhost:11300'])
|
17
|
+
end
|
18
|
+
|
9
19
|
it "should default to localhost:11300" do
|
10
|
-
Beanstalk::Pool.should_receive(:new).with(['localhost:11300'])
|
11
|
-
JobQueue::BeanstalkAdapter.new
|
20
|
+
Beanstalk::Pool.should_receive(:new).with(['localhost:11300'], "default").and_return @pool
|
21
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
22
|
+
JobQueue.put('test')
|
12
23
|
end
|
13
24
|
|
14
|
-
it "should
|
15
|
-
Beanstalk::Pool.should_receive(:new).with(['12.34.56.78:12345'])
|
16
|
-
JobQueue::BeanstalkAdapter.new(:
|
25
|
+
it "should accept one beanstalk instance" do
|
26
|
+
Beanstalk::Pool.should_receive(:new).with(['12.34.56.78:12345'], 'default').and_return(@pool)
|
27
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new(:hosts => '12.34.56.78:12345')
|
28
|
+
JobQueue.put('test')
|
17
29
|
end
|
18
|
-
end
|
19
30
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
@
|
25
|
-
|
31
|
+
it "should allow multiple beanstalk instances" do
|
32
|
+
Beanstalk::Pool.should_receive(:new).with([
|
33
|
+
'12.34.56.78:12345',
|
34
|
+
'87.65.43.21:54321'
|
35
|
+
], 'default').and_return(@pool)
|
36
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
|
37
|
+
:hosts => ['12.34.56.78:12345', '87.65.43.21:54321']
|
38
|
+
})
|
39
|
+
JobQueue.put('test')
|
26
40
|
end
|
27
|
-
|
28
|
-
@job.should == "hello"
|
29
41
|
end
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
42
|
+
|
43
|
+
describe "when connecting to one instance" do
|
44
|
+
before :all do
|
45
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should write onto queue and fetch stuff back off" do
|
49
|
+
JobQueue.put("hello")
|
50
|
+
|
51
|
+
should_not_timeout {
|
52
|
+
JobQueue.subscribe do |job|
|
53
|
+
@job = job
|
54
|
+
throw :stop
|
55
|
+
end
|
56
|
+
}
|
57
|
+
|
58
|
+
@job.should == "hello"
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should output message if error raised in job" do
|
62
|
+
JobQueue.put("hello")
|
63
|
+
JobQueue.put("hello2")
|
64
|
+
|
65
|
+
JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
|
66
|
+
|
67
|
+
should_not_timeout {
|
68
|
+
index = 0
|
69
|
+
JobQueue.subscribe do |job|
|
70
|
+
index +=1
|
71
|
+
raise 'foo' if index == 1
|
72
|
+
throw :stop
|
73
|
+
end
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should use error_report block if supplied" do
|
78
|
+
JobQueue.put("hello")
|
79
|
+
JobQueue.put("hello2")
|
80
|
+
|
81
|
+
error_report = Proc.new do |job, e|
|
82
|
+
JobQueue.logger.error "Yikes that broke matey!"
|
83
|
+
end
|
84
|
+
|
85
|
+
JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
|
86
|
+
|
87
|
+
should_not_timeout {
|
88
|
+
index = 0
|
89
|
+
JobQueue.subscribe(:error_report => error_report) do |job|
|
90
|
+
index +=1
|
91
|
+
raise 'foo' if index == 1
|
92
|
+
throw :stop
|
93
|
+
end
|
94
|
+
}
|
95
|
+
end
|
96
|
+
|
97
|
+
it "should put jobs onto a named queue and only read off that queue" do
|
98
|
+
JobQueue.put("hello", :queue => "test")
|
99
|
+
lambda {
|
100
|
+
Timeout.timeout(0.1) do
|
101
|
+
JobQueue.subscribe(:queue => "foo") do |job|
|
102
|
+
throw :stop
|
103
|
+
end
|
104
|
+
end
|
105
|
+
}.should raise_error(Timeout::Error)
|
106
|
+
should_not_timeout {
|
107
|
+
JobQueue.subscribe(:queue => "test") do |body|
|
108
|
+
body.should == 'hello'
|
109
|
+
throw :stop
|
110
|
+
end
|
111
|
+
}
|
41
112
|
end
|
42
113
|
end
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
114
|
+
|
115
|
+
describe "when connecting to multiple instances" do
|
116
|
+
before :all do
|
117
|
+
JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
|
118
|
+
:hosts => ['localhost:10001', 'localhost:10002']
|
119
|
+
})
|
120
|
+
end
|
121
|
+
|
122
|
+
it "should be possible to put jobs" do
|
123
|
+
JobQueue.put('test')
|
124
|
+
JobQueue.subscribe do |job|
|
125
|
+
job.should == 'test'
|
126
|
+
throw :stop
|
127
|
+
end
|
49
128
|
end
|
50
129
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
130
|
+
# TODO: This test is brittle.
|
131
|
+
it "should be possible to retrieve all jobs supplied" do
|
132
|
+
# Put some jobs on the queue
|
133
|
+
jobs = []
|
134
|
+
(1..8).each do |i|
|
135
|
+
body = i
|
136
|
+
JobQueue.put("#{body}")
|
137
|
+
jobs << body
|
138
|
+
end
|
139
|
+
|
140
|
+
should_not_timeout(3.5) {
|
141
|
+
JobQueue.subscribe do |job|
|
142
|
+
jobs.delete job.to_i
|
143
|
+
throw :stop if jobs.empty?
|
144
|
+
end
|
145
|
+
}
|
146
|
+
end
|
147
|
+
|
148
|
+
it "should not log any errors when reserve times out" do
|
149
|
+
JobQueue.logger.should_not_receive(:error)
|
150
|
+
begin
|
151
|
+
Timeout::timeout(1.5) do
|
152
|
+
JobQueue.subscribe { |job| }
|
153
|
+
end
|
154
|
+
rescue Timeout::Error
|
155
|
+
#Do nothing - timeout expected
|
156
|
+
end
|
58
157
|
end
|
59
158
|
end
|
60
159
|
end
|
160
|
+
|
161
|
+
def should_not_timeout(timeout = 0.1)
|
162
|
+
lambda {
|
163
|
+
Timeout.timeout(timeout) do
|
164
|
+
yield
|
165
|
+
end
|
166
|
+
}.should_not raise_error(Timeout::Error)
|
167
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mloughran-job_queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Martyn Loughran
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-
|
12
|
+
date: 2009-06-11 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -19,20 +19,19 @@ executables: []
|
|
19
19
|
|
20
20
|
extensions: []
|
21
21
|
|
22
|
-
extra_rdoc_files:
|
23
|
-
|
24
|
-
- README.markdown
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
25
24
|
files:
|
26
|
-
- LICENSE
|
27
25
|
- README.markdown
|
28
|
-
- Rakefile
|
29
26
|
- VERSION.yml
|
30
|
-
- lib/job_queue
|
27
|
+
- lib/job_queue
|
28
|
+
- lib/job_queue/adapters
|
31
29
|
- lib/job_queue/adapters/amqp_adapter.rb
|
32
30
|
- lib/job_queue/adapters/beanstalk_adapter.rb
|
33
31
|
- lib/job_queue/adapters/test_adapter.rb
|
34
32
|
- lib/job_queue/adapters/verbose_adapter.rb
|
35
33
|
- lib/job_queue/job_queue.rb
|
34
|
+
- lib/job_queue.rb
|
36
35
|
- spec/amqp_adapter_spec.rb
|
37
36
|
- spec/beanstalk_adapter_spec.rb
|
38
37
|
- spec/job_queue_spec.rb
|
@@ -43,6 +42,7 @@ has_rdoc: true
|
|
43
42
|
homepage: http://github.com/mloughran/job_queue
|
44
43
|
post_install_message:
|
45
44
|
rdoc_options:
|
45
|
+
- --inline-source
|
46
46
|
- --charset=UTF-8
|
47
47
|
require_paths:
|
48
48
|
- lib
|
@@ -65,10 +65,5 @@ rubygems_version: 1.2.0
|
|
65
65
|
signing_key:
|
66
66
|
specification_version: 2
|
67
67
|
summary: JobQueue means you don't have to worry about your queue any more!
|
68
|
-
test_files:
|
69
|
-
|
70
|
-
- spec/beanstalk_adapter_spec.rb
|
71
|
-
- spec/job_queue_spec.rb
|
72
|
-
- spec/spec_helper.rb
|
73
|
-
- spec/test_adapter_spec.rb
|
74
|
-
- spec/verbose_adapter_spec.rb
|
68
|
+
test_files: []
|
69
|
+
|
data/LICENSE
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
Copyright (c) 2009 Martyn Loughran
|
2
|
-
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
-
a copy of this software and associated documentation files (the
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
9
|
-
the following conditions:
|
10
|
-
|
11
|
-
The above copyright notice and this permission notice shall be
|
12
|
-
included in all copies or substantial portions of the Software.
|
13
|
-
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
require 'rake'
|
2
|
-
|
3
|
-
begin
|
4
|
-
require 'jeweler'
|
5
|
-
Jeweler::Tasks.new do |s|
|
6
|
-
s.name = "job_queue"
|
7
|
-
s.summary = %Q{JobQueue means you don't have to worry about your queue any more!}
|
8
|
-
s.email = "me@mloughran.com"
|
9
|
-
s.homepage = "http://github.com/mloughran/job_queue"
|
10
|
-
s.description = "JobQueue means you don't have to worry about your queue any more!"
|
11
|
-
s.authors = ["Martyn Loughran"]
|
12
|
-
end
|
13
|
-
rescue LoadError
|
14
|
-
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
15
|
-
end
|
16
|
-
|
17
|
-
require 'rake/rdoctask'
|
18
|
-
Rake::RDocTask.new do |rdoc|
|
19
|
-
rdoc.rdoc_dir = 'rdoc'
|
20
|
-
rdoc.title = 'job_queue'
|
21
|
-
rdoc.options << '--line-numbers' << '--inline-source'
|
22
|
-
rdoc.rdoc_files.include('README*')
|
23
|
-
rdoc.rdoc_files.include('lib/**/*.rb')
|
24
|
-
end
|
25
|
-
|
26
|
-
require 'spec/rake/spectask'
|
27
|
-
Spec::Rake::SpecTask.new(:spec) do |t|
|
28
|
-
t.libs << 'lib' << 'spec'
|
29
|
-
t.spec_files = FileList['spec/**/*_spec.rb']
|
30
|
-
end
|
31
|
-
|
32
|
-
Spec::Rake::SpecTask.new(:rcov) do |t|
|
33
|
-
t.libs << 'lib' << 'spec'
|
34
|
-
t.spec_files = FileList['spec/**/*_spec.rb']
|
35
|
-
t.rcov = true
|
36
|
-
end
|
37
|
-
|
38
|
-
task :default => :spec
|