mloughran-job_queue 0.0.4 → 0.0.5

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 4
-:major: 0
 :minor: 0
+:patch: 5
+:major: 0

data/lib/job_queue/adapters/beanstalk_adapter.rb

@@ -2,25 +2,35 @@ require 'beanstalk-client'
 
 class JobQueue::BeanstalkAdapter
   def initialize(options = {})
-    host = options[:host] || 'localhost'
-    port = options[:port] || 11300
-    @beanstalk = Beanstalk::Pool.new(["#{host}:#{port}"])
+    @hosts = options[:hosts] || 'localhost:11300'
   end
   
-  def put(string)
-    @beanstalk.put(string)
+  def put(string, queue, priority)
+    beanstalk_pool(queue).put(string)
   end
   
-  def subscribe(error_report, &block)
+  def subscribe(error_report, queue, &block)
+    pool = Beanstalk::Pool.new([@hosts].flatten, queue)
     loop do
       begin
-        job = @beanstalk.reserve
+        job = pool.reserve(1)
         JobQueue.logger.info "Beanstalk received #{job.body}"
-        yield job.body
-        job.delete
-      rescue => e
-        error_report.call(job.body, e)
+        begin
+          yield job.body
+        rescue => e
+          error_report.call(job.body, e)
+          job.delete
+        end
+      rescue Beanstalk::TimedOut
+        # Do nothing - retry to reseve (from another host?)
       end
     end
   end
+  
+  def beanstalk_pool(queue)
+    @beanstalk_pools ||= {}
+    @beanstalk_pools[queue] ||= begin
+      Beanstalk::Pool.new([@hosts].flatten, queue)
+    end
+  end
 end

data/lib/job_queue/job_queue.rb

@@ -30,23 +30,27 @@ class JobQueue
     end
   end
   
-  def self.put(string)
-    adapter.put(string)
+  def self.put(string, options = {})
+    queue = options[:queue] || 'default'
+    priority = options[:priority] || 50
+    adapter.put(string, queue, priority)
   end
   
-  def self.subscribe(error_report = nil, &block)
-    catch :stop do
-      error_report ||= Proc.new do |job, e|
+  def self.subscribe(options = {}, &block)
+    queue = options[:queue] || 'default'
+    error_report = options[:error_report] || begin
+      Proc.new do |job_body, e|
         JobQueue.logger.error \
           "Job failed\n" \
           "==========\n" \
-          "Job content: #{job.inspect}\n" \
+          "Job content: #{job_body.inspect}\n" \
           "Exception: #{e.message}\n" \
           "#{e.backtrace.join("\n")}\n" \
           "\n"
       end
-      
-      adapter.subscribe(error_report, &block)
+    end
+    catch :stop do
+      adapter.subscribe(error_report, queue, &block)
     end
   end
 end

data/spec/beanstalk_adapter_spec.rb

@@ -1,60 +1,167 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
 describe JobQueue::BeanstalkAdapter do
-  before :all do
-    JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+  before :each do
+    system "beanstalkd -p 10001 -d"
+    system "beanstalkd -p 10002 -d"
+    system "beanstalkd -p 11300 -d"
+  end
+
+  after :each do
+    system "killall beanstalkd"
   end
 
   describe '#new' do
+    before(:each) do
+      @pool = Beanstalk::Pool.new(['localhost:11300'])
+    end
+
     it "should default to localhost:11300" do
-      Beanstalk::Pool.should_receive(:new).with(['localhost:11300'])
-      JobQueue::BeanstalkAdapter.new
+      Beanstalk::Pool.should_receive(:new).with(['localhost:11300'], "default").and_return @pool
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+      JobQueue.put('test')
     end
 
-    it "should use options provided" do
-      Beanstalk::Pool.should_receive(:new).with(['12.34.56.78:12345'])
-      JobQueue::BeanstalkAdapter.new(:host => '12.34.56.78', :port => 12345)
+    it "should accept one beanstalk instance" do
+      Beanstalk::Pool.should_receive(:new).with(['12.34.56.78:12345'], 'default').and_return(@pool)
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new(:hosts => '12.34.56.78:12345')
+      JobQueue.put('test')
     end
-  end
 
-  it "should write onto queue and fetch stuff back off" do
-    JobQueue.put("hello")
-    
-    JobQueue.subscribe do |job|
-      @job = job
-      throw :stop
+    it "should allow multiple beanstalk instances" do
+      Beanstalk::Pool.should_receive(:new).with([
+        '12.34.56.78:12345',
+        '87.65.43.21:54321'
+      ], 'default').and_return(@pool)
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
+        :hosts => ['12.34.56.78:12345', '87.65.43.21:54321']
+      })
+      JobQueue.put('test')
     end
-    
-    @job.should == "hello"
   end
-  
-  it "should output message if error raised in job" do
-    JobQueue.put("hello")
-    
-    JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
-    
-    index = 0
-    JobQueue.subscribe do |job|
-      index +=1
-      raise 'foo' if index == 1
-      throw :stop
+
+  describe "when connecting to one instance" do
+    before :all do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+    end
+
+    it "should write onto queue and fetch stuff back off" do
+      JobQueue.put("hello")
+
+      should_not_timeout {
+        JobQueue.subscribe do |job|
+          @job = job
+          throw :stop
+        end
+      }
+
+      @job.should == "hello"
+    end
+
+    it "should output message if error raised in job" do
+      JobQueue.put("hello")
+      JobQueue.put("hello2")
+
+      JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
+
+      should_not_timeout {
+        index = 0
+        JobQueue.subscribe do |job|
+          index +=1
+          raise 'foo' if index == 1
+          throw :stop
+        end
+      }
+    end
+
+    it "should use error_report block if supplied" do
+      JobQueue.put("hello")
+      JobQueue.put("hello2")
+
+      error_report = Proc.new do |job, e|
+        JobQueue.logger.error "Yikes that broke matey!"
+      end
+
+      JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
+
+      should_not_timeout {
+        index = 0
+        JobQueue.subscribe(:error_report => error_report) do |job|
+          index +=1
+          raise 'foo' if index == 1
+          throw :stop
+        end
+      }
+    end
+
+    it "should put jobs onto a named queue and only read off that queue" do
+      JobQueue.put("hello", :queue => "test")
+      lambda {
+        Timeout.timeout(0.1) do
+          JobQueue.subscribe(:queue => "foo") do |job|
+            throw :stop
+          end
+        end
+      }.should raise_error(Timeout::Error)
+      should_not_timeout {
+        JobQueue.subscribe(:queue => "test") do |body|
+          body.should == 'hello'
+          throw :stop
+        end
+      }
     end
   end
-  
-  it "should use error_report block if supplied" do
-    JobQueue.put("hello")
-    
-    error_report = Proc.new do |job, e|
-      JobQueue.logger.error "Yikes that broke matey!"
+
+  describe "when connecting to multiple instances" do
+    before :all do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
+        :hosts => ['localhost:10001', 'localhost:10002']
+      })
+    end
+
+    it "should be possible to put jobs" do
+      JobQueue.put('test')
+      JobQueue.subscribe do |job|
+        job.should == 'test'
+        throw :stop
+      end
     end
     
-    JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
-    
-    index = 0
-    JobQueue.subscribe(error_report) do |job|
-      index +=1
-      raise 'foo' if index == 1
-      throw :stop
+    # TODO: This test is brittle.
+    it "should be possible to retrieve all jobs supplied" do
+      # Put some jobs on the queue
+      jobs = []
+      (1..8).each do |i|
+        body = i
+        JobQueue.put("#{body}")
+        jobs << body
+      end
+
+      should_not_timeout(3.5) {
+        JobQueue.subscribe do |job|
+          jobs.delete job.to_i
+          throw :stop if jobs.empty?
+        end
+      }
+    end
+
+    it "should not log any errors when reserve times out" do
+      JobQueue.logger.should_not_receive(:error)
+      begin
+        Timeout::timeout(1.5) do
+          JobQueue.subscribe { |job| }
+        end
+      rescue Timeout::Error
+        #Do nothing - timeout expected
+      end
     end
   end
 end
+
+def should_not_timeout(timeout = 0.1)
+  lambda {
+    Timeout.timeout(timeout) do
+      yield
+    end
+  }.should_not raise_error(Timeout::Error)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mloughran-job_queue
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors: 
 - Martyn Loughran
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-16 00:00:00 -07:00
+date: 2009-06-11 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -19,20 +19,19 @@ executables: []
 
 extensions: []
 
-extra_rdoc_files: 
-- LICENSE
-- README.markdown
+extra_rdoc_files: []
+
 files: 
-- LICENSE
 - README.markdown
-- Rakefile
 - VERSION.yml
-- lib/job_queue.rb
+- lib/job_queue
+- lib/job_queue/adapters
 - lib/job_queue/adapters/amqp_adapter.rb
 - lib/job_queue/adapters/beanstalk_adapter.rb
 - lib/job_queue/adapters/test_adapter.rb
 - lib/job_queue/adapters/verbose_adapter.rb
 - lib/job_queue/job_queue.rb
+- lib/job_queue.rb
 - spec/amqp_adapter_spec.rb
 - spec/beanstalk_adapter_spec.rb
 - spec/job_queue_spec.rb
@@ -43,6 +42,7 @@ has_rdoc: true
 homepage: http://github.com/mloughran/job_queue
 post_install_message: 
 rdoc_options: 
+- --inline-source
 - --charset=UTF-8
 require_paths: 
 - lib
@@ -65,10 +65,5 @@ rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: JobQueue means you don't have to worry about your queue any more!
-test_files: 
-- spec/amqp_adapter_spec.rb
-- spec/beanstalk_adapter_spec.rb
-- spec/job_queue_spec.rb
-- spec/spec_helper.rb
-- spec/test_adapter_spec.rb
-- spec/verbose_adapter_spec.rb
+test_files: []
+

data/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2009 Martyn Loughran
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,38 +0,0 @@
-require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |s|
-    s.name = "job_queue"
-    s.summary = %Q{JobQueue means you don't have to worry about your queue any more!}
-    s.email = "me@mloughran.com"
-    s.homepage = "http://github.com/mloughran/job_queue"
-    s.description = "JobQueue means you don't have to worry about your queue any more!"
-    s.authors = ["Martyn Loughran"]
-  end
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
-end
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = 'job_queue'
-  rdoc.options << '--line-numbers' << '--inline-source'
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |t|
-  t.libs << 'lib' << 'spec'
-  t.spec_files = FileList['spec/**/*_spec.rb']
-end
-
-Spec::Rake::SpecTask.new(:rcov) do |t|
-  t.libs << 'lib' << 'spec'
-  t.spec_files = FileList['spec/**/*_spec.rb']
-  t.rcov = true
-end
-
-task :default => :spec