mloughran-job_queue 0.0.5 → 0.0.6

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 0
-:patch: 5
+:patch: 6
 :major: 0

data/lib/job_queue/adapters/beanstalk_adapter.rb

@@ -4,33 +4,87 @@ class JobQueue::BeanstalkAdapter
   def initialize(options = {})
     @hosts = options[:hosts] || 'localhost:11300'
   end
-  
-  def put(string, queue, priority)
-    beanstalk_pool(queue).put(string)
+
+  def put(string, queue, priority, ttr)
+    ttr = ttr.floor #rounding because Beanstalk doesnt accept float numbers
+    raise JobQueue::ArgumentError, "TTR must be greater than 1" if ttr < 2
+
+    delay = 0
+    job_info = beanstalk_pool(queue).put_and_report_conn \
+      string, priority, delay, ttr
+    "#{job_info[:host]}_#{job_info[:id]}"
+  rescue Beanstalk::NotConnected
+    raise JobQueue::NoConnectionAvailable
   end
-  
-  def subscribe(error_report, queue, &block)
+
+  def subscribe(error_report, cleanup_task, queue, &block)
     pool = Beanstalk::Pool.new([@hosts].flatten, queue)
     loop do
       begin
         job = pool.reserve(1)
+        time_left = job.stats["time-left"]
         JobQueue.logger.info "Beanstalk received #{job.body}"
-        begin
+        Timeout::timeout([time_left - 1, 1].max) do
           yield job.body
-        rescue => e
-          error_report.call(job.body, e)
+        end
+        job.delete
+      rescue Timeout::Error
+        cleanup_task.call(job.body)
+        JobQueue.logger.warn "Job timed out"
+        begin
           job.delete
+        rescue Beanstalk::NotFoundError
+          JobQueue.logger.error "Job timed out and could not be deleted"
         end
       rescue Beanstalk::TimedOut
         # Do nothing - retry to reseve (from another host?)
+      rescue => e
+        if job
+          error_report.call(job.body, e)
+          begin
+            job.delete
+          rescue Beanstalk::NotFoundError
+            JobQueue.logger.error "Job failed but could not be deleted"
+          end
+        else
+          JobQueue.logger.error "Unhandled exception: #{e.message}\n" \
+            "#{e.backtrace.join("\n")}\n"
+        end
       end
     end
   end
-  
-  def beanstalk_pool(queue)
+
+  def job_stats(job_id)
+    host, id = job_id.split('_')
+    beanstalk_pool.job_stats(id).select { |k, v| k == host }[0][1]
+  rescue Beanstalk::NotFoundError
+    nil
+  end
+
+  def beanstalk_pool(queue='default')
     @beanstalk_pools ||= {}
     @beanstalk_pools[queue] ||= begin
       Beanstalk::Pool.new([@hosts].flatten, queue)
     end
   end
+
+  module BeanstalkExtension
+    def put_and_report_conn(body, pri=65536, delay=0, ttr=120)
+      send_to_rand_conn_and_report(:put, body, pri, delay, ttr)
+    end
+
+    def send_to_rand_conn_and_report(*args)
+      connect()
+      retry_wrap{
+        conn = pick_connection
+        {:host => conn.addr, :id => call_wrap(conn, *args)}
+      }
+    end
+
+    def job_stats(id)
+      make_hash(send_to_all_conns(:job_stats, id))
+    end
+  end
 end
+
+Beanstalk::Pool.send(:include, JobQueue::BeanstalkAdapter::BeanstalkExtension)

data/lib/job_queue/job_queue.rb

@@ -29,28 +29,39 @@ class JobQueue
       end
     end
   end
-  
+
   def self.put(string, options = {})
     queue = options[:queue] || 'default'
     priority = options[:priority] || 50
-    adapter.put(string, queue, priority)
+    ttr = options[:ttr] || 60
+    adapter.put(string, queue, priority, ttr)
   end
-  
+
   def self.subscribe(options = {}, &block)
     queue = options[:queue] || 'default'
-    error_report = options[:error_report] || begin
-      Proc.new do |job_body, e|
-        JobQueue.logger.error \
-          "Job failed\n" \
-          "==========\n" \
-          "Job content: #{job_body.inspect}\n" \
-          "Exception: #{e.message}\n" \
-          "#{e.backtrace.join("\n")}\n" \
-          "\n"
-      end
+    error_report = options[:error_report] || Proc.new do |job_body, e|
+      JobQueue.logger.error \
+        "Job failed\n" \
+        "==========\n" \
+        "Job content: #{job_body.inspect}\n" \
+        "Exception: #{e.message}\n" \
+        "#{e.backtrace.join("\n")}\n" \
+        "\n"
     end
+    cleanup_task = options[:cleanup] || lambda {}
     catch :stop do
-      adapter.subscribe(error_report, queue, &block)
+      adapter.subscribe(error_report, cleanup_task, queue, &block)
     end
   end
+
+  # Returns a hash of info (exact details dependent on adapter)
+  def self.job_stats(job_id)
+    adapter.job_stats(job_id)
+  end
+
+  class NoConnectionAvailable < RuntimeError
+  end
+
+  class ArgumentError < ::ArgumentError
+  end
 end

data/spec/beanstalk_adapter_spec.rb

@@ -40,6 +40,137 @@ describe JobQueue::BeanstalkAdapter do
     end
   end
 
+  describe "put" do
+    before :all do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+    end
+
+    it "should return the job id" do
+      job_id = JobQueue.put("hello 1")
+      job_id.should == "localhost:11300_1"
+    end
+
+    it "should assign job priority" do
+      jobs = ["1","2","3"]
+      JobQueue.put(jobs[2], :priority => 3)
+      JobQueue.put(jobs[1], :priority => 2)
+      JobQueue.put(jobs[0], :priority => 1)
+
+      jobs_received = []
+      should_not_timeout(0.5) {
+        index = 0
+        JobQueue.subscribe do |job_body|
+          index += 1
+          jobs_received << job_body
+          throw :stop if index == 3
+        end
+      }
+
+      jobs_received.should == jobs
+    end
+
+    it "should be able to retrieve job stats by id" do
+      job_id = JobQueue.put("hello 1")
+      job_id.should == "localhost:11300_1"
+      JobQueue.put("hello 2")
+      stats = JobQueue.job_stats("localhost:11300_1")
+
+      stats["id"].should == 1
+      stats["tube"].should == "default"
+    end
+
+    it "should raise error when no connections exist" do
+      system "killall beanstalkd"
+      lambda {
+        JobQueue.put('test')
+      }.should raise_error(JobQueue::NoConnectionAvailable)
+    end
+
+    it "should succeed when one connection fails" do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new({
+        :hosts => ['localhost:10001', 'localhost:666']
+      })
+      10.times{ job_id = JobQueue.put("hello 1")}
+    end
+
+    it "should raise an error if a ttr of < 2 is specified" do
+      lambda {
+        JobQueue.put('test', :ttr => 1.9)
+      }.should raise_error(JobQueue::ArgumentError)
+
+      lambda {
+        JobQueue.put('test', :ttr => 2)
+      }.should_not raise_error(JobQueue::ArgumentError)
+    end
+  end
+
+  describe "subscribe" do
+    before :all do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+    end
+
+    it "should delete a job once it has been succesfully excecuted" do
+      job_id = JobQueue.put('testdeleted')
+      JobQueue.put('foo')
+      index = 0
+      JobQueue.subscribe do |body|
+        index += 1
+        throw :stop if index == 2
+      end
+      JobQueue.job_stats(job_id).should be_nil
+    end
+
+    it "should report and error and delete the job if a job times out" do
+      job_id = JobQueue.put("job1", :ttr => 2)
+      JobQueue.put('test')
+
+      JobQueue.logger.should_receive(:warn).with("Job timed out")
+
+      index = 0
+      JobQueue.subscribe do |body|
+        index += 1
+        throw :stop if index == 2
+        sleep 2.2
+      end
+
+      JobQueue.job_stats(job_id).should be_nil
+    end
+
+    it "should allow a client to cleanup if a job times out" do
+      JobQueue.put('jobcleanup', :ttr => 2)
+      JobQueue.put('test')
+
+      cleanup = nil
+
+      index = 0
+      JobQueue.subscribe(:cleanup => lambda { |job| FileUtils.rm(job) }) do |body|
+        file = File.open(body, 'w')
+        file << "hello"
+        file.flush
+
+        index += 1
+        throw :stop if index == 2
+        sleep 2.2
+      end
+
+      File.exists?('jobcleanup').should be_false
+    end
+  end
+
+  describe "job_stats" do
+    before :all do
+      JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+    end
+
+    it "should gracefully deal with jobs where connection no longer exists" do
+      JobQueue.job_stats("localhost:11305_1").should be_nil
+    end
+
+    it "should gracefully deal with jobs where job doesn't exist" do
+      JobQueue.job_stats("localhost:11300_1").should be_nil
+    end
+  end
+
   describe "when connecting to one instance" do
     before :all do
       JobQueue.adapter = JobQueue::BeanstalkAdapter.new
@@ -126,7 +257,7 @@ describe JobQueue::BeanstalkAdapter do
         throw :stop
       end
     end
-    
+
     # TODO: This test is brittle.
     it "should be possible to retrieve all jobs supplied" do
       # Put some jobs on the queue

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mloughran-job_queue
 version: !ruby/object:Gem::Version 
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors: 
 - Martyn Loughran
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-11 00:00:00 -07:00
+date: 2009-06-12 00:00:00 -07:00
 default_executable: 
 dependencies: []