tom_queue 0.0.1.dev

data/spec/tom_queue/deferred_work/deferred_work_set_spec.rb

@@ -0,0 +1,134 @@
+require 'tom_queue/helper'
+
+describe TomQueue::DeferredWorkSet do
+  let(:set) { TomQueue::DeferredWorkSet.new }
+
+  it "should be creatable" do
+    set.should be_a(TomQueue::DeferredWorkSet)
+  end
+
+  it "should allow work to be scheduled" do
+    set.schedule(Time.now + 0.2, "something")
+    set.schedule(Time.now + 0.3, "else")
+    set.size.should == 2
+  end
+
+  describe "earliest" do
+
+    it "should return nil if there is no work in the set" do
+      set.earliest.should be_nil
+    end
+
+    it "should return the only item if there is one item in the set" do
+      work = double("Work")
+      set.schedule( Time.now + 0.3, work)
+      set.earliest.should == work
+    end
+
+    it "should return the item in the set with the lowest run_at value" do
+      set.schedule( Time.now + 0.2, work1 = double("Work") )
+      set.schedule( Time.now + 0.1, work2 = double("Work") )
+      set.schedule( Time.now + 0.3, work3 = double("Work") )
+      set.earliest.should == work2
+    end
+
+  end
+
+  describe "pop" do
+
+    it "should return nil when the timeout expires" do
+      set.pop(0.1).should be_nil
+    end
+
+    it "should block for the timeout value if there is no work in the queue" do
+      start_time = Time.now
+      set.pop(0.1)
+      Time.now.should > start_time + 0.1
+    end
+
+    it "should block until the earliest work in the set" do
+      start_time = Time.now
+      set.schedule(start_time + 1.5, "work")
+      set.schedule(start_time + 0.1, "work")
+      set.pop(10)
+      Time.now.should > start_time + 0.1
+      Time.now.should < start_time + 0.2
+    end
+
+    it "should return immediately if tehre is work scheduled in the past" do
+      set.schedule(Time.now - 0.1, "work")
+      set.pop(10).should == "work"
+    end
+
+    it "should have removed the returned work from the set" do
+      set.schedule(Time.now - 0.1, "work")
+      set.pop(10)
+      set.size.should == 0
+    end
+
+    it "should return old work in temporal order" do
+      set.schedule(Time.now - 0.1, "work2")
+      set.schedule(Time.now - 0.2, "work1")
+      set.pop(10).should == "work1"
+      set.pop(10).should == "work2"
+    end
+
+    it "should return the earliest work" do
+      start_time = Time.now
+      set.schedule(start_time + 0.1, "work")
+      set.pop(10).should == "work"
+    end
+
+    it "should return immediately if it is interrupted by an external thread" do
+      Thread.new { sleep 0.1; set.interrupt }
+      start_time = Time.now
+      set.schedule(start_time + 1.5, "work")
+      set.schedule(start_time + 5, "work")
+      set.pop(10)
+      Time.now.should > start_time + 0.1
+      Time.now.should < start_time + 0.2
+    end
+
+    it "should block until the earliest work, even if earlier work is added after the block" do
+      start_time = Time.now
+      Thread.new do
+        sleep 0.1
+        set.schedule(start_time + 0.2, "early")
+      end
+      set.schedule(start_time + 1.5, "late")
+      set.pop(10)
+      Time.now.should > start_time + 0.2
+      Time.now.should < start_time + 0.3
+    end
+
+    it "should raise an exception if two threads try to block on the same work set" do
+      Thread.new do
+        set.pop(1)
+      end
+      sleep 0.1
+      lambda {
+        set.pop(1)
+      }.should raise_exception(/another thread is already blocked/)
+    end
+
+    it "should not get deferred items caught outside the cache" do
+      start_time = Time.now
+      50.times { |i| set.schedule(start_time+0.1+i*0.001, "bulk") }
+      set.schedule(start_time+0.2, "missing")
+
+      50.times { set.pop(1).should == "bulk" }
+
+      set.schedule(start_time+0.3, "final")
+      set.pop(1).should == "missing"
+      set.pop(1).should == "final"
+    end
+
+    it "should not delete all elements with the same run_at" do
+      the_time = Time.now + 0.1
+      set.schedule(the_time, "work-1")
+      set.schedule(the_time, "work-2")
+      2.times.collect { set.pop(1) }.sort.should == ["work-1", "work-2"]
+    end
+  end
+
+end

data/spec/tom_queue/delayed_job/delayed_job_integration_spec.rb

@@ -0,0 +1,155 @@
+require 'tom_queue/helper'
+require 'tom_queue/delayed_job'
+
+describe Delayed::Job, "integration spec", :timeout => 10 do
+    
+  class TestJobClass
+    cattr_accessor :perform_hook
+    
+    @@flunk_count = 0
+    cattr_accessor :flunk_count 
+    
+    @@asplode_count = 0
+    cattr_accessor :asplode_count
+
+    def initialize(name)
+      @name = name
+    end
+
+    def perform
+      @@perform_hook && @@perform_hook.call(@name)
+
+      if @@asplode_count > 0
+        @@asplode_count -= 1
+        Thread.exit
+      end
+
+      if @@flunk_count > 0
+        @@flunk_count -= 1
+        raise RuntimeError, "Failed to run job"
+      end
+    end
+
+    def reschedule_at(time, attempts)
+      time + 0.5
+    end
+
+  end
+
+  let(:job_name) { "Job-#{Time.now.to_f}" }
+
+  before do
+    # Clean-slate ...
+    TomQueue.default_prefix = "test-#{Time.now.to_f}"
+    TomQueue::DelayedJob.apply_hook!
+    Delayed::Job.delete_all
+    Delayed::Job.class_variable_set(:@@tomqueue_manager, nil)
+    
+    # Keep track of how many times the job is run
+    @called = []
+    TestJobClass.perform_hook = lambda { |name| @called << name }
+
+    # Reset the flunk count
+    TestJobClass.flunk_count = 0
+  end
+
+  it "should actually be using the queue" do
+    Delayed::Job.enqueue(TestJobClass.new(job_name))
+
+    Delayed::Job.tomqueue_manager.queues[TomQueue::NORMAL_PRIORITY].status[:message_count].should == 1
+  end
+
+  it "should integrate with Delayed::Worker" do
+    Delayed::Job.enqueue(TestJobClass.new(job_name))
+
+    Delayed::Worker.new.work_off(1).should == [1, 0] # 1 success, 0 failed
+    @called.first.should == job_name
+  end
+
+  it "should still back-off jobs" do
+    Delayed::Job.enqueue(TestJobClass.new(job_name))
+    TestJobClass.flunk_count = 1
+
+    Benchmark.realtime { 
+      Delayed::Worker.new.work_off(1).should == [0, 1]
+      Delayed::Worker.new.work_off(1).should == [1, 0]
+    }.should > 0.5
+  end
+
+  it "should support run_at" do
+    Benchmark.realtime {
+      Delayed::Job.enqueue(TestJobClass.new("job1"), :run_at => Time.now + 0.5)
+      Delayed::Job.enqueue(TestJobClass.new("job2"), :run_at => Time.now + 0.1)
+      Delayed::Worker.new.work_off(2).should == [2, 0]
+    }.should > 0.1
+    @called.should == ["job2", "job1"]
+  end
+
+  it "should support job priorities" do
+    TomQueue::DelayedJob.priority_map[0] = TomQueue::NORMAL_PRIORITY
+    TomQueue::DelayedJob.priority_map[1] = TomQueue::HIGH_PRIORITY
+    Delayed::Job.enqueue(TestJobClass.new("low1"), :priority => 0)
+    Delayed::Job.enqueue(TestJobClass.new("low2"), :priority => 0)
+    Delayed::Job.enqueue(TestJobClass.new("high"), :priority => 1)
+    Delayed::Job.enqueue(TestJobClass.new("low3"), :priority => 0)
+    Delayed::Job.enqueue(TestJobClass.new("low4"), :priority => 0)
+    Delayed::Worker.new.work_off(5)
+    @called.should == ["high", "low1", "low2", "low3", "low4"]
+  end
+
+  it "should not run a failed job" do
+    logfile = Tempfile.new('logfile')
+    TomQueue.logger = Logger.new(logfile.path)
+    Delayed::Job.delete_all
+    # this will send the notification
+    job = "Hello".delay.to_s
+
+    # now make the job look like it has failed
+    job.attempts = 0
+    job.failed_at = Time.now
+    job.last_error = "Some error"
+    job.save
+
+    job.should be_failed
+
+    Delayed::Job.tomqueue_republish
+
+    # The job should get ignored for both runs
+    Delayed::Worker.new.work_off(1)
+    Delayed::Worker.new.work_off(1)
+    
+    # And, since it never got run, it should still exist!
+    Delayed::Job.find_by_id(job.id).should_not be_nil
+    # And it should have been noisy, too.
+    File.read(logfile.path).should =~ /Received notification for failed job #{job.id}/
+  end
+
+  # it "should re-run the job once max_run_time is reached if, say, a worker crashes" do
+  #   Delayed::Worker.max_run_time = 2
+
+  #   job = Delayed::Job.enqueue(TestJobClass.new("work"))
+
+  #   # This thread will be abruptly terminated mid-job
+  #   TestJobClass.asplode_count = 1
+  #   lock_stale_time = Time.now.to_f + Delayed::Worker.max_run_time
+  #   Thread.new { Delayed::Worker.new.work_off(1) }.join
+
+  #   # This will shutdown the various channels, which should result in the message being
+  #   # returned to the broker.
+  #   Delayed::Job.tomqueue_manager.setup_amqp!
+
+  #   # Make sure the job is still locked
+  #   job.reload
+  #   job.locked_at.should_not be_nil
+  #   job.locked_by.should_not be_nil
+    
+  #   # Now wait for the max_run_time, which is artificially low
+  #   while Delayed::Job.find_by_id(job.id)
+  #     Delayed::Worker.new.work_off(1)
+  #   end
+
+  #   # Ensure the worker blocked until the job's original lock was actually stale.
+  #   Time.now.to_f.should > lock_stale_time.to_f
+  # end
+
+end

data/spec/tom_queue/delayed_job/delayed_job_spec.rb

@@ -0,0 +1,818 @@
+require 'tom_queue/helper'
+
+describe TomQueue, "once hooked" do
+
+  let(:job) { Delayed::Job.create! }
+  let(:new_job) { Delayed::Job.new }
+
+
+  it "should set the Delayed::Worker sleep delay to 0" do
+    # This makes sure the Delayed::Worker loop spins around on
+    # an empty queue to block on TomQueue::QueueManager#pop, so
+    # the job will start as soon as we receive a push from RMQ
+    Delayed::Worker.sleep_delay.should == 0
+  end
+
+  describe "TomQueue::DelayedJob::Job" do
+    it "should use the TomQueue job as the Delayed::Job" do
+      Delayed::Job.should == TomQueue::DelayedJob::Job
+    end
+
+    it "should be a subclass of ::Delayed::Backend::ActiveRecord::Job" do
+      TomQueue::DelayedJob::Job.superclass.should == ::Delayed::Backend::ActiveRecord::Job
+    end
+  end
+
+  describe "Delayed::Job.tomqueue_manager" do
+    it "should return a TomQueue::QueueManager instance" do
+      Delayed::Job.tomqueue_manager.should be_a(TomQueue::QueueManager)
+    end
+
+    it "should have used the default prefix configured" do
+      Delayed::Job.tomqueue_manager.prefix.should == TomQueue.default_prefix
+    end
+
+    it "should return the same object on subsequent calls" do
+      Delayed::Job.tomqueue_manager.should == Delayed::Job.tomqueue_manager
+    end
+
+    it "should be reset by rspec (1)" do
+      TomQueue.default_prefix = "foo"
+      Delayed::Job.tomqueue_manager.prefix.should == "foo"
+    end
+
+    it "should be reset by rspec (2)" do
+      TomQueue.default_prefix = "bar"
+      Delayed::Job.tomqueue_manager.prefix.should == "bar"
+    end
+  end
+
+  describe "Delayed::Job#tomqueue_digest" do
+
+    it "should return a different value when the object is saved" do
+      first_digest = job.tomqueue_digest
+      job.update_attributes(:run_at => Time.now + 10.seconds)
+      job.tomqueue_digest.should_not == first_digest
+    end
+
+    it "should return the same value, regardless of the time zone (regression)" do
+      ActiveRecord::Base.time_zone_aware_attributes = true
+      old_zone, Time.zone = Time.zone, "Hawaii"
+
+      job = Delayed::Job.create!
+      first_digest = job.tomqueue_digest
+
+      Time.zone = "Auckland"
+
+      job = Delayed::Job.find(job.id)
+      job.tomqueue_digest.should == first_digest
+
+      Time.zone = old_zone
+    end
+  end
+
+  describe "Delayed::Job#tomqueue_payload" do
+ 
+    let(:payload) { JSON.load(job.tomqueue_payload)}
+ 
+    it "should return a hash" do
+      payload.should be_a(Hash)
+    end
+
+    it "should contain the job id" do
+      payload['delayed_job_id'].should == job.id
+    end
+
+    it "should contain the current updated_at timestamp (with second-level precision)" do
+      payload['delayed_job_updated_at'].should == job.updated_at.iso8601(0)
+    end
+
+    it "should contain the digest after saving" do
+      payload['delayed_job_digest'].should == job.tomqueue_digest
+    end
+  end
+
+  describe "Delayed::Job#tomqueue_publish" do
+
+    it "should return nil" do
+      job.tomqueue_publish.should be_nil
+    end
+
+    it "should raise an exception if it is called on an unsaved job" do
+      TomQueue.exception_reporter = double("SilentExceptionReporter", :notify => nil)
+      lambda {
+        Delayed::Job.new.tomqueue_publish
+      }.should raise_exception(ArgumentError, /cannot publish an unsaved Delayed::Job/)
+    end
+
+    describe "when it is called on a persisted job" do
+
+      before do
+        job # create the job first so we don't trigger the expectation twice
+
+        @called = false
+        Delayed::Job.tomqueue_manager.should_receive(:publish) do |payload, opts|
+          @called = true
+          @payload = payload
+          @opts = opts
+        end
+      end
+
+      it "should call publish on the queue manager" do
+        job.tomqueue_publish
+        @called.should be_true
+      end
+
+      describe "job priority" do
+        before do
+          TomQueue::DelayedJob.priority_map[-10] = TomQueue::BULK_PRIORITY
+          TomQueue::DelayedJob.priority_map[10]  = TomQueue::HIGH_PRIORITY
+        end
+
+        it "should map the priority of the job to the TomQueue priority" do
+          new_job.priority = -10
+          new_job.save
+          @opts[:priority].should == TomQueue::BULK_PRIORITY
+        end
+
+        describe "if an unknown priority value is used" do
+          before do
+            new_job.priority = 99
+          end
+
+          it "should default the priority to TomQueue::NORMAL_PRIORITY" do
+            new_job.save
+            @opts[:priority].should == TomQueue::NORMAL_PRIORITY
+          end
+
+          it "should log a warning" do
+            TomQueue.logger.should_receive(:warn)
+            new_job.save
+          end
+        end
+      end
+
+      describe "run_at value" do
+
+        it "should use the job's :run_at value by default" do
+          job.tomqueue_publish
+          @opts[:run_at].should == job.run_at
+        end
+
+        it "should use the run_at value provided if provided by the caller" do
+          the_time = Time.now + 10.seconds
+          job.tomqueue_publish(the_time)
+          @opts[:run_at].should == the_time
+        end
+
+      end
+
+      describe "the payload" do
+
+        before { job.stub(:tomqueue_payload => "PAYLOAD") }
+
+        it "should be the return value from #tomqueue_payload" do
+          job.tomqueue_publish
+          @payload.should == "PAYLOAD"
+        end
+      end
+    end
+
+    describe "if an exception is raised during the publish" do
+      let(:exception) { RuntimeError.new("Bugger. Dropped the ball, sorry.") }
+
+      before do
+        TomQueue.exception_reporter = double("SilentExceptionReporter", :notify => nil)
+        Delayed::Job.tomqueue_manager.should_receive(:publish).and_raise(exception)
+      end
+
+      it "should not be raised out to the caller" do
+        lambda { new_job.save }.should_not raise_exception
+      end
+
+      it "should notify the exception reporter" do
+        TomQueue.exception_reporter.should_receive(:notify).with(exception)
+        new_job.save
+      end
+
+      it "should do nothing if the exception reporter is nil" do
+        TomQueue.exception_reporter = nil
+        lambda { new_job.save }.should_not raise_exception
+      end
+      
+      it "should log an error message to the log" do
+        TomQueue.logger.should_receive(:error)
+        new_job.save
+      end
+    end
+  end
+
+  describe "publish callbacks in Job lifecycle" do
+
+    it "should allow Mock::ExpectationFailed exceptions to escape the callback" do
+      TomQueue.logger = Logger.new("/dev/null")
+      TomQueue.exception_reporter = nil
+      Delayed::Job.tomqueue_manager.should_receive(:publish).with("spurious arguments").once
+      lambda {
+        job.update_attributes(:run_at => Time.now + 5.seconds)
+      }.should raise_exception(RSpec::Mocks::MockExpectationError)
+
+      Delayed::Job.tomqueue_manager.publish("spurious arguments") # do this, otherwise it will fail
+    end    
+
+    it "should not publish a message if the job has a non-nil failed_at" do
+      job.should_not_receive(:tomqueue_publish)
+      job.update_attributes(:failed_at => Time.now)
+    end
+
+    it "should be called after create when there is no explicit transaction" do
+      new_job.should_receive(:tomqueue_publish).with(no_args)
+      new_job.save!
+    end
+
+    it "should be called after update when there is no explicit transaction" do
+      job.should_receive(:tomqueue_publish).with(no_args)
+      job.run_at = Time.now + 10.seconds
+      job.save!
+    end
+
+    it "should be called after commit, when a record is saved" do
+      new_job.stub(:tomqueue_publish) { @called = true }
+      Delayed::Job.transaction do
+        new_job.save!
+
+        @called.should be_nil
+      end
+      @called.should be_true
+    end
+
+    it "should be called after commit, when a record is updated" do
+      job.stub(:tomqueue_publish) { @called = true }
+      Delayed::Job.transaction do
+        job.run_at = Time.now + 10.seconds
+        job.save!
+        @called.should be_nil
+      end
+
+      @called.should be_true
+    end
+
+    it "should not be called when a record is destroyed" do
+      job.should_not_receive(:tomqueue_publish)
+      job.destroy
+    end
+
+    it "should not be called by a destroy in a transaction" do
+      job.should_not_receive(:tomqueue_publish)
+      Delayed::Job.transaction { job.destroy }
+    end
+
+    it "should not be called if the update transaction is rolled back" do
+      job.stub(:tomqueue_publish) { @called = true }
+
+      Delayed::Job.transaction do
+        job.run_at = Time.now + 10.seconds
+        job.save!
+        raise ActiveRecord::Rollback
+      end
+      @called.should be_nil
+    end
+
+    it "should not be called if the create transaction is rolled back" do
+      job.should_not_receive(:tomqueue_publish)
+
+      Delayed::Job.transaction do
+        new_job.save!
+        raise ActiveRecord::Rollback
+      end
+      @called.should be_nil
+    end
+  end
+
+  describe "Delayed::Job.tomqueue_republish method" do
+    before { Delayed::Job.delete_all }
+
+    it "should exist" do
+      Delayed::Job.respond_to?(:tomqueue_republish).should be_true
+    end
+
+    it "should return nil" do
+      Delayed::Job.tomqueue_republish.should be_nil
+    end
+
+    it "should call #tomqueue_publish on all DB records" do
+      10.times { Delayed::Job.create! }
+
+      Delayed::Job.tomqueue_manager.queues[TomQueue::NORMAL_PRIORITY].purge
+      queue = Delayed::Job.tomqueue_manager.queues[TomQueue::NORMAL_PRIORITY]
+      queue.message_count.should == 0
+
+      Delayed::Job.tomqueue_republish
+      queue.message_count.should == 10
+    end
+
+    it "should work with ActiveRecord scopes" do
+      first_ids = 10.times.collect { Delayed::Job.create!.id }
+      second_ids = 7.times.collect { Delayed::Job.create!.id }
+
+      Delayed::Job.tomqueue_manager.queues[TomQueue::NORMAL_PRIORITY].purge
+      queue = Delayed::Job.tomqueue_manager.queues[TomQueue::NORMAL_PRIORITY]
+      queue.message_count.should == 0
+
+      Delayed::Job.where('id IN (?)', second_ids).tomqueue_republish
+      queue.message_count.should == 7
+    end
+
+  end
+
+  describe "Delayed::Job.acquire_locked_job" do
+    let(:time) { Delayed::Job.db_time_now }
+    before { Delayed::Job.stub(:db_time_now => time) }
+    
+    let(:job) { Delayed::Job.create! }
+    let(:worker) { Delayed::Worker.new }
+
+    # make sure the job exists!
+    before { job }
+
+    subject { Delayed::Job.acquire_locked_job(job.id, worker, &@block) }
+
+    describe "when the job doesn't exist" do
+      before { job.destroy }
+
+      it "should return nil" do
+        subject.should be_nil
+      end
+
+      it "should not yield if a block is provided" do
+        @block = lambda { |value| @called = true}
+        subject
+        @called.should be_nil
+      end
+    end
+
+    describe "when the job exists" do
+
+      it "should hold an explicit DB lock whilst performing the lock" do
+        pending("Only possible when using mysql2 adapter (ADAPTER=mysql environment)") unless ActiveRecord::Base.connection.class.to_s == "ActiveRecord::ConnectionAdapters::Mysql2Adapter"
+
+        # Ok, fudge a second parallel connection to MySQL
+        second_connection = ActiveRecord::Base.connection.dup
+        second_connection.reconnect!
+        ActiveRecord::Base.connection.reset!
+
+        # Assert we have separate connections
+        second_connection.select("SELECT connection_id() as id;").first["id"].should_not ==
+          ActiveRecord::Base.connection.select("SELECT connection_id() as id;").first["id"]
+
+        # This is called in a thread when the transaction is open to query the job, store the response
+        # and the time when the response comes back
+        parallel_query = lambda do |job_id|
+          begin
+            # Simulate another worker performing a SELECT ... FOR UPDATE request
+            @query_result = second_connection.select("SELECT * FROM delayed_jobs WHERE id=#{job_id} LIMIT 1 FOR UPDATE").first
+            @query_returned_at = Time.now.to_f
+          rescue
+            puts "Failed #{$!.inspect}"
+          end
+        end
+
+        # When we have the transaction open, we emit a query on a parallel thread and check the timing
+        @block = lambda do |job|
+          @thread = Thread.new(job.id, &parallel_query)
+          sleep 0.25
+          @leaving_transaction_at = Time.now.to_f
+          true
+        end
+        
+        # Kick it all off !
+        subject
+          
+        @thread.join
+
+        # now make sure the parallel thread blocked until the transaction returned
+        @query_returned_at.should > @leaving_transaction_at
+
+        # make sure the returned record showed the lock
+        @query_result["locked_at"].should_not be_nil
+        @query_result["locked_by"].should_not be_nil
+      end
+
+      describe "when the job is marked as failed" do
+        let(:failed_time) { Time.now - 10 }
+
+        before do
+          job.update_attribute(:failed_at, failed_time)
+        end
+
+        it "should return nil" do
+          subject.should be_nil
+        end
+
+        it "should not modify the failed_at value" do
+          subject
+          job.reload
+          job.failed_at.to_i.should == failed_time.to_i
+        end
+
+        it "should not lock the job" do
+          subject
+          job.reload
+          job.locked_by.should be_nil
+          job.locked_at.should be_nil
+        end
+      end
+
+      describe "when the notification is delivered too soon" do
+        
+        before do
+          actual_time = Delayed::Job.db_time_now
+          Delayed::Job.stub(:db_time_now => actual_time - 10)
+        end
+
+        it "should return nil" do
+          subject.should be_nil
+        end
+
+        it "should re-post a notification" do
+          Delayed::Job.tomqueue_manager.should_receive(:publish) do |payload, args|
+            args[:run_at].to_i.should == job.run_at.to_i
+          end
+          subject
+        end
+
+        it "should not lock the job" do
+          subject
+          job.reload
+          job.locked_by.should be_nil
+          job.locked_at.should be_nil
+        end
+
+      end
+
+      describe "when the job is not locked" do
+        
+        it "should acquire the lock fields on the job" do
+          subject
+          job.reload
+          job.locked_at.to_i.should == time.to_i
+          job.locked_by.should == worker.name
+        end
+
+        it "should return the job object" do
+          subject.should be_a(Delayed::Job)
+          subject.id.should == job.id
+        end
+
+        it "should yield the job to the block if present" do
+          @block = lambda { |value| @called = value}
+          subject
+          @called.should be_a(Delayed::Job)
+          @called.id.should == job.id
+        end
+
+        it "should not have locked the job when the block is called" do
+          @block = lambda { |job| @called = [job.id, job.locked_at, job.locked_by]; true }
+          subject
+          @called.should == [job.id, nil, nil]
+        end
+
+        describe "if the supplied block returns true" do
+          before { @block = lambda { |_| true } }
+
+          it "should lock the job" do
+            subject
+            job.reload
+            job.locked_at.to_i.should == time.to_i
+            job.locked_by.should == worker.name
+          end
+
+          it "should return the job" do
+            subject.should be_a(Delayed::Job)
+            subject.id.should == job.id
+          end
+        end
+
+        describe "if the supplied block returns false" do
+          before { @block = lambda { |_| false } }
+
+          it "should not lock the job" do
+            subject
+            job.reload
+            job.locked_at.should be_nil
+            job.locked_by.should be_nil
+          end
+
+          it "should return nil" do
+            subject.should be_nil
+          end
+        end
+      end
+
+      describe "when the job is locked with a valid lock" do
+
+        before do
+          @old_locked_by = job.locked_by = "some worker"
+          @old_locked_at = job.locked_at = Time.now
+          job.save!
+        end
+
+        it "should not yield to a block if provided" do
+          @called = false
+          @block = lambda { |_| @called = true}
+          subject
+          @called.should be_false
+        end
+
+        it "should return false" do
+          subject.should be_false
+        end
+
+        it "should not change the lock" do
+          subject
+          job.reload
+          job.locked_by.should == @old_locked_by
+          job.locked_at.to_i.should == @old_locked_at.to_i
+        end
+
+      end
+
+      describe "when the job is locked with a stale lock" do
+        before do
+          @old_locked_by = job.locked_by = "some worker"
+          @old_locked_at = job.locked_at = (Time.now - Delayed::Worker.max_run_time - 1)
+          job.save!
+        end
+
+        it "should return the job" do
+          subject.should be_a(Delayed::Job)
+          subject.id.should == job.id
+        end
+
+        it "should update the lock" do
+          subject
+          job.reload
+          job.locked_at.should_not == @old_locked_at
+          job.locked_by.should_not == @old_locked_by
+        end
+
+        # This is tricky - if we have a stale lock, the job object
+        # will have been updated by the first worker, so the digest will
+        # now be invalid (since updated_at will have changed)
+        #
+        # So, we don't yield, we just presume we're carrying on from where
+        # a previous worker left off and don't try and validate the job any
+        # further.
+        it "should not yield the block if supplied" do
+          @called = false
+          @block = lambda { |_| @called = true}
+          subject
+          @called.should be_false
+        end
+      end
+
+    end
+  end
+
+  describe "Delayed::Job.reserve - return the next job" do
+    let(:job)     { Delayed::Job.create! }
+    let(:worker)  { double("Worker", :name => "Worker-Name-#{Time.now.to_f}") }
+    let(:payload) { job.tomqueue_payload }
+    let(:work)    { double("Work", :payload => payload, :ack! => nil) }
+    
+    subject { Delayed::Job.reserve(worker) }
+
+    before do
+      Delayed::Job.tomqueue_manager.stub(:pop => work)
+    end
+
+    it "should call pop on the queue manager" do
+      Delayed::Job.tomqueue_manager.should_receive(:pop)
+      
+      subject
+    end
+
+    describe "signal handling" do
+      it "should allow signal handlers during the pop" do
+        Delayed::Worker.raise_signal_exceptions = false
+        Delayed::Job.tomqueue_manager.should_receive(:pop) do
+          Delayed::Worker.raise_signal_exceptions.should be_true
+          work
+        end
+        Delayed::Job.reserve(worker)
+      end
+
+      it "should reset the signal handler var after the pop" do
+        Delayed::Worker.raise_signal_exceptions = false
+        subject
+        Delayed::Worker.raise_signal_exceptions.should == false
+      end
+
+      it "should reset the signal handler var even if it's already true" do
+        Delayed::Worker.raise_signal_exceptions = true
+        subject
+        Delayed::Worker.raise_signal_exceptions.should == true
+      end
+
+      it "should allow exceptions to escape the function" do
+        Delayed::Job.tomqueue_manager.should_receive(:pop) do
+          raise SignalException, "INT"
+        end
+        lambda { subject }.should raise_exception(SignalException)
+      end
+    end
+
+    describe "if a nil message is popped" do
+      before { Delayed::Job.tomqueue_manager.stub(:pop=>nil) }
+
+      it "should return nil" do
+        subject.should be_nil
+      end
+
+      it "should sleep for a second to avoid potentially tight loops" do
+        start_time = Time.now
+        subject
+        (Time.now - start_time).should > 1.0
+      end
+    end
+
+    describe "if the work payload doesn't cleanly JSON decode" do
+      before do
+        TomQueue.logger = Logger.new("/dev/null")
+        TomQueue.exception_reporter = nil
+      end
+
+      let(:payload) { "NOT JSON!!1" }
+      
+      it "should report an exception" do
+        TomQueue.exception_reporter = mock("Reporter", :notify => nil)
+        TomQueue.exception_reporter.should_receive(:notify).with(instance_of(JSON::ParserError))
+        subject
+      end
+
+      it "should be happy if no exception reporter is set" do
+        TomQueue.exception_reporter = nil
+        subject
+      end
+
+      it "should ack the message" do
+        work.should_receive(:ack!)
+        subject
+      end
+
+      it "should log an error" do
+        TomQueue.logger.should_receive(:error)
+        subject
+      end
+
+      it "should return nil" do
+        subject.should be_nil
+      end
+    end
+
+    it "should call acquire_locked_job with the job_id and the worker" do
+      Delayed::Job.should_receive(:acquire_locked_job).with(job.id, worker)
+      subject      
+    end
+
+    it "should attach a block to the call to acquire_locked_job" do
+      def stub_implementation(job_id, worker, &block)
+        block.should_not be_nil
+      end
+      Delayed::Job.should_receive(:acquire_locked_job, &method(:stub_implementation)).and_return(job)
+      subject
+    end
+
+    describe "the block provided to acquire_locked_job" do
+      before do |test|
+        def stub_implementation(job_id, worker, &block)
+          @block = block
+        end
+        Delayed::Job.should_receive(:acquire_locked_job, &method(:stub_implementation)).and_return(job)
+      end
+
+      it "should return true if the digest in the message payload matches the job" do
+        subject
+        @block.call(job).should be_true
+      end
+
+      it "should return false if the digest in the message payload doesn't match the job" do
+        subject
+        job.touch(:updated_at)
+        @block.call(job).should be_true
+      end
+
+      it "should return true if there is no digest in the payload object" do
+        work.stub(:payload => JSON.dump(JSON.load(payload).merge("delayed_job_digest" => nil)))
+        subject
+        @block.call(job).should be_true
+      end
+
+    end
+
+    describe "when acquire_locked_job returns the job object" do
+      # A.K.A We have a locked job!
+
+      let(:returned_job) { Delayed::Job.find(job.id) }
+      before { Delayed::Job.stub(:acquire_locked_job => returned_job) }
+
+      it "should not ack the message" do
+        work.should_not_receive(:ack!)
+        subject
+      end
+
+      it "should return the Job object" do
+        subject.should == returned_job
+      end
+
+      it "should associate the message object with the job" do
+        subject.tomqueue_work.should == work
+      end
+
+    end
+
+    describe "when acquire_locked_job returns false" do
+      # A.K.A The lock is held by another worker.
+      #  - we post a notification to re-try after the max_run_time
+
+      before do
+        job.locked_at = Delayed::Job.db_time_now - 10
+        job.locked_by = "foobar"
+        job.save!
+        Delayed::Job.stub(:acquire_locked_job => false)
+      end
+
+      it "should ack the message" do
+        work.should_receive(:ack!)
+        subject
+      end
+
+      it "should publish a notification for after the max-run-time of the job" do
+        Delayed::Job.tomqueue_manager.should_receive(:publish) do |payload, opts|
+          opts[:run_at].to_i.should == job.locked_at.to_i + Delayed::Worker.max_run_time + 1
+        end
+        subject
+      end
+
+      it "should return nil" do
+        subject.should be_nil
+      end
+
+    end
+
+    describe "when acquire_locked_job returns nil" do
+      # A.K.A The job doesn't exist anymore!
+      #  - we're done!
+
+      before { Delayed::Job.stub(:acquire_locked_job => nil) }
+
+      it "should ack the message" do
+        work.should_receive(:ack!)
+        subject
+      end
+
+      it "should return nil" do
+        subject.should be_nil
+      end
+
+    end
+
+
+  end
+
+  describe "Job#invoke_job" do
+    let(:payload) { double("DelayedJobPayload", :perform => nil)}
+    let(:job) { Delayed::Job.create!(:payload_object=>payload) }
+
+    it "should perform the job" do
+      payload.should_receive(:perform)
+      job.invoke_job
+    end
+
+    it "should not have a problem if tomqueue_work is nil" do
+      job.tomqueue_work = nil
+      job.invoke_job
+    end
+
+    describe "if there is a tomqueue work object set on the object" do
+      let(:work_object) { double("WorkObject", :ack! => nil)}
+      before { job.tomqueue_work = work_object}
+
+      it "should call ack! on the work object after the job has been invoked" do
+        payload.should_receive(:perform).ordered
+        work_object.should_receive(:ack!).ordered
+        job.invoke_job
+      end
+
+      it "should call ack! on the work object if an exception is raised" do
+        payload.should_receive(:perform).ordered.and_raise(RuntimeError, "OMG!!!11")
+        work_object.should_receive(:ack!).ordered
+        lambda {
+          job.invoke_job
+        }.should raise_exception(RuntimeError, "OMG!!!11")
+      end
+    end
+  end
+
+
+end