backburner 1.6.1 → 1.8.0

data/test/workers/simple_worker_test.rb

@@ -256,6 +256,35 @@ describe "Backburner::Workers::Simple module" do
       assert_equal true, $worker_success
     end
 
+    it "should round a fractional retry delay up to whole seconds" do
+      conn = beanstalk_connection
+      default_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+      [
+        [TestConfigurableRetryJob, 5,   lambda { |min_retry_delay, num_retries| 1.5 }, 2], # a retry_delay_proc returning a Float
+        [TestConfigurableRetryJob, 0.5, default_proc, 1], # a Float retry_delay
+        [TestFloatRetryDelayJob,   5,   default_proc, 2]  # a class's Float queue_retry_delay (1.5)
+      ].each do |job_class, retry_delay, retry_delay_proc, expected|
+        clear_jobs!('foo.bar')
+        Backburner.configure do |config|
+          config.max_job_retries = 1
+          config.retry_delay = retry_delay
+          config.retry_delay_proc = retry_delay_proc
+        end
+        id = @worker_class.enqueue(job_class, [5], :queue => 'foo.bar')[:id]
+        out = silenced(5) do
+          worker = @worker_class.new('foo.bar')
+          worker.prepare
+          worker.work_one_job
+        end
+        assert_match(/attempt 1 of 2, retrying in #{expected}s/, out)
+        stats = conn.jobs.find(id).stats
+        assert_equal [expected, 'delayed'], [stats.delay, stats.state], "#{job_class} with retry_delay #{retry_delay}"
+      end
+    ensure
+      conn.close if conn
+      clear_jobs!('foo.bar')
+    end
+
     it "should support event hooks without retry" do
       $hooked_fail_count = 0
       clear_jobs!('foo.bar.events')
@@ -332,6 +361,205 @@ describe "Backburner::Workers::Simple module" do
       refute_match(/HookFailError/, out)
     end # stopping perform
 
+    it "should delete a job whose before_perform hook stopped it" do
+      clear_jobs!('foo.bar.events.retry3')
+      worker = @worker_class.new('foo.bar.events.retry3')
+      silenced(2) do
+        HookedObjectBeforePerformFail.async(:queue => 'foo.bar.events.retry3').foo(10)
+        worker.prepare
+        worker.work_one_job
+      end
+      stats = worker.connection.tubes['demo.test.foo.bar.events.retry3'].stats
+      assert_equal 0, stats.current_jobs_reserved, "the aborted job is still reserved"
+      assert_equal 0, stats.current_jobs_ready + stats.current_jobs_delayed + stats.current_jobs_buried
+    end # stopping perform deletes
+
+    it "should keep working when the TTR ran out before the job could be buried" do
+      clear_jobs!("foo.bar.ttr")
+      @worker_class.enqueue TestOverTTRJob, [], :queue => "foo.bar.ttr"
+      worker = @worker_class.new('foo.bar.ttr')
+      out = silenced(6) do
+        worker.prepare
+        worker.work_one_job
+      end
+      assert_match(/TestOverTTRJob.* hit 1s timeout/, out)
+      assert_match(/TestOverTTRJob was no longer reserved/, out)
+      # beanstalkd took the job back when its TTR ran out and hands it out again
+      pop_one_job("foo.bar.ttr") { |job, body| assert_equal "TestOverTTRJob", body["class"]; job.delete }
+    end
+
+    it "should keep working when the TTR ran out before the job could be retried" do
+      Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
+      clear_jobs!("foo.bar.ttr")
+      @worker_class.enqueue TestOverTTRJob, [], :queue => "foo.bar.ttr"
+      worker = @worker_class.new('foo.bar.ttr')
+      out = silenced(6) do
+        worker.prepare
+        worker.work_one_job
+      end
+      assert_match(/TestOverTTRJob was no longer reserved/, out)
+      pop_one_job("foo.bar.ttr") { |job, body| assert_equal "TestOverTTRJob", body["class"]; job.delete }
+    end
+
+    it "should keep working when the connection is lost before the failed job could be retried or buried" do
+      clear_jobs!("foo.bar.lost")
+      @worker_class.enqueue TestFailJob, [1, 2], :queue => "foo.bar.lost"
+      worker = @worker_class.new('foo.bar.lost')
+      errors = []
+      Backburner.configure { |config| config.on_error = lambda { |e| errors << e } }
+      # beanstalkd went away while the job ran: asking for its stats (to retry or bury it)
+      # raises NotConnected
+      Beaneater::Job.any_instance.stubs(:ttr).returns(10)
+      Beaneater::Job.any_instance.stubs(:stats).raises(Beaneater::NotConnected)
+      out = silenced(2) do
+        worker.prepare
+        worker.work_one_job
+      end
+      assert_match(/TestFailJob was no longer reserved/, out)
+      assert_equal [RuntimeError], errors.map(&:class)
+    ensure
+      Beaneater::Job.any_instance.unstub(:stats)
+      Beaneater::Job.any_instance.unstub(:ttr)
+      Backburner.configure { |config| config.on_error = nil }
+    end
+
+    describe "with a job whose class can't be found" do
+      before do
+        clear_jobs!("foo.bar.unknown")
+        @errors = []
+        Backburner.configure { |config| config.on_error = lambda { |e| @errors << e } }
+        @worker = @worker_class.new('foo.bar.unknown')
+      end
+
+      after do
+        Backburner.configure { |config| config.on_error = nil }
+        clear_jobs!("foo.bar.unknown")
+      end
+
+      def put_job(class_name, ttr)
+        body = Backburner.configuration.job_serializer_proc.call(:class => class_name, :args => [1])
+        @worker.connection.tubes["demo.test.foo.bar.unknown"].put(body, :ttr => ttr)
+      end
+
+      def tube_stats
+        @worker.connection.tubes["demo.test.foo.bar.unknown"].stats
+      end
+
+      it "should release it for its TTR instead of burying it, and keep working" do
+        put_job("TestRemovedJob", 30)
+        out = silenced(2) do
+          @worker.prepare
+          @worker.work_one_job
+        end
+        assert_match(/Exception Backburner::Job::JobNotFound -> TestRemovedJob/, out)
+        assert_match(/TestRemovedJob: class not found; released for 30s/, out)
+        assert_equal [], @errors, "on_error isn't called for a class this worker lacks"
+        assert_equal 0, tube_stats.current_jobs_buried
+        assert_equal 1, tube_stats.current_jobs_delayed
+        assert_equal 30, @worker.connection.tubes["demo.test.foo.bar.unknown"].peek(:delayed).stats.delay
+      end
+
+      it "should leave it for a worker that has its class, however many retries it is allowed" do
+        Backburner.configure { |config| config.max_job_retries = 0 }
+        put_job("TestRolloutJob", 2)
+        out = silenced(8) do
+          @worker.prepare
+          @worker.work_one_job # this worker doesn't have the class yet
+          Object.const_set(:TestRolloutJob, Class.new { def self.perform(x); $worker_test_count += x; end })
+          @worker.work_one_job # now it does: the job comes back after its TTR
+        end
+        assert_match(/TestRolloutJob: class not found; released for 2s/, out)
+        assert_match(/Completed TestRolloutJob/, out)
+        assert_equal 1, $worker_test_count
+        assert_equal 0, tube_stats.current_jobs_buried
+      ensure
+        Object.send(:remove_const, :TestRolloutJob) if Object.const_defined?(:TestRolloutJob)
+      end
+
+      it "should release a job whose class an autoloader can't load" do
+        put_job("AutoloadingDemo::TestJobGone", 30)
+        out = silenced(2) do
+          @worker.prepare
+          @worker.work_one_job
+        end
+        assert_match(/AutoloadingDemo::TestJobGone: class not found; released for 30s/, out)
+        assert_equal [0, 1], [tube_stats.current_jobs_buried, tube_stats.current_jobs_delayed]
+      end
+
+      [["TestBrokenNameJob", "NameError"], ["TestBrokenRequireJob", "LoadError"], ["TestBrokenNoMethodJob", "NoMethodError"],
+       ["TestBrokenRuntimeJob", "ArgumentError"], ["TestSameNameJob", "NameError"], ["TestBrokenSyntaxJob", "SyntaxError"],
+       ["not a class", "NameError"], ["TestBrokenTimeoutJob", "Timeout::Error"]].each do |class_name, error|
+        it "should retry and bury a job whose class fails to load or can't exist (#{class_name}), calling on_error" do
+          Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
+          put_job(class_name, 30)
+          out = silenced(4) do
+            @worker.prepare
+            2.times { @worker.work_one_job }
+          end
+          assert_match(/Exception Backburner::Job::JobNotFound -> #{class_name} \(#{error}: /, out)
+          assert_match(/attempt 1 of 2, retrying in 0s/, out)
+          assert_match(/attempt 2 of 2, burying/, out)
+          refute_match(/class not found/, out)
+          assert_equal [Backburner::Job::JobNotFound] * 2, @errors.map(&:class)
+          assert_equal 1, tube_stats.current_jobs_buried
+        end
+      end
+
+      %w(Backburner::TestGoneJob TestGoneNamespace::TestJob TestNoFileJob).each do |class_name|
+        it "should release a job whose class is missing (#{class_name})" do
+          put_job(class_name, 30)
+          out = silenced(2) do
+            @worker.prepare
+            @worker.work_one_job
+          end
+          assert_match(/#{class_name}: class not found; released for 30s/, out)
+          assert_equal [], @errors
+          assert_equal [0, 1], [tube_stats.current_jobs_buried, tube_stats.current_jobs_delayed]
+        end
+      end
+
+      it "should release it for at most MAX_CLASS_NOT_FOUND_DELAY seconds" do
+        put_job("TestRemovedJob", 3600)
+        out = silenced(2) do
+          @worker.prepare
+          @worker.work_one_job
+        end
+        assert_match(/TestRemovedJob: class not found; released for 60s/, out)
+        assert_equal 60, @worker.connection.tubes["demo.test.foo.bar.unknown"].peek(:delayed).stats.delay
+      end
+
+      it "should not call on_error when its TTR ran out before it could be released" do
+        put_job("TestRemovedJob", 1)
+        def @worker.reserve_job(*args); job = super; job.ttr; sleep 2.2; job; end # its TTR runs out first
+        out = silenced(6) do
+          @worker.prepare
+          @worker.work_one_job
+        end
+        assert_match(/TestRemovedJob was no longer reserved/, out)
+        assert_equal [], @errors
+        assert_equal [0, 1], [tube_stats.current_jobs_buried, tube_stats.current_jobs_ready]
+      end
+    end # unknown class
+
+    it "should keep working when the on_error handler raises" do
+      clear_jobs!("foo.bar.onerror")
+      Backburner.configure { |config| config.on_error = lambda { |e| raise SocketError, "error reporter unreachable" } }
+      @worker_class.enqueue TestFailJob, [1, 2], :queue => "foo.bar.onerror"
+      @worker_class.enqueue TestJob, [1, 2], :queue => "foo.bar.onerror"
+      worker = @worker_class.new('foo.bar.onerror')
+      out = silenced(4) do
+        worker.prepare
+        2.times { worker.work_one_job }
+      end
+      assert_match(/Finished TestFailJob.*attempt 1 of 1, burying/, out)
+      assert_match(/The on_error handler failed for TestFailJob: Exception SocketError -> error reporter unreachable/, out)
+      assert_equal 1, worker.connection.tubes["demo.test.foo.bar.onerror"].stats.current_jobs_buried
+      assert_equal 3, $worker_test_count
+    ensure
+      Backburner.configure { |config| config.on_error = nil }
+      clear_jobs!("foo.bar.onerror")
+    end
+
     it "should use the connection given as an argument" do
       worker = @worker_class.new('foo.bar')
       connection = mock('connection')
@@ -347,4 +575,48 @@ describe "Backburner::Workers::Simple module" do
       end
     end
   end # work_one_job
+
+  describe "when beanstalkd is down for longer than the reconnect window" do
+    before do
+      pick_beanstalkd_port
+      @templogger = Templogger.new('/tmp')
+      @beanstalk_url = Backburner.configuration.beanstalk_url
+      Backburner.configure do |config|
+        config.beanstalk_url = "beanstalk://127.0.0.1:#{@port}"
+        config.logger = @templogger.logger
+      end
+      start_beanstalkd
+      $worker_test_count = 0
+    end
+
+    after do
+      @worker_thread.kill if @worker_thread
+      stop_beanstalkd
+      if @templogger
+        Backburner.configure do |config|
+          config.beanstalk_url = @beanstalk_url
+          config.logger = nil
+        end
+        @templogger.close
+      end
+    end
+
+    it "should reconnect, watch its tubes again and resume working" do
+      worker = @worker_class.new('foo.bar.outage')
+      worker.prepare
+      @worker_thread = Thread.new { loop { worker.work_one_job } }
+      sleep 0.5 # let the worker block in reserve
+
+      stop_beanstalkd
+      # beaneater's own reconnect attempts fail, then Connection#ensure_connected! gives up too
+      wait_for { @templogger.body.scan(/Exception Beaneater::NotConnected/).size >= 2 }
+      start_beanstalkd
+      @worker_class.enqueue TestJob, [1, 2], :queue => 'foo.bar.outage'
+      wait_for(10) { $worker_test_count == 3 }
+
+      errors = @templogger.body.scan(/ERROR -- /).size
+      assert_operator errors, :<, 20, "the worker logged #{errors} errors (busy loop?)"
+      assert_equal 3, $worker_test_count
+    end
+  end # beanstalkd outage
 end # Worker

data/test/workers/threading_worker_test.rb

@@ -1,6 +1,7 @@
 require File.expand_path('../../test_helper', __FILE__)
 require File.expand_path('../../fixtures/test_jobs', __FILE__)
 require File.expand_path('../../fixtures/hooked', __FILE__)
+require File.expand_path('../../fixtures/test_queue_settings', __FILE__)
 
 describe "Backburner::Workers::Threading worker" do
   before do
@@ -51,6 +52,21 @@ describe "Backburner::Workers::Threading worker" do
     end
   end # process_tube_names
 
+  describe "for process_tube_options method" do
+    it "should take a shared tube's limit from the class that owns the tube, not from subclasses" do
+      worker = @worker_class.new(%w(tube-limits tube-limits-testtubelimitslambdachildjob))
+      assert_equal({
+        "demo.test.tube-limits" => { :threads => 5 },
+        "demo.test.tube-limits-testtubelimitslambdachildjob" => { :threads => 4 }
+      }, worker.instance_variable_get("@tubes_data"))
+    end
+
+    it "should fill in a shared tube's limit from a subclass when the tube's class sets none" do
+      worker = @worker_class.new(%w(tube-few-limits))
+      assert_equal({ "demo.test.tube-few-limits" => { :threads => 3 } }, worker.instance_variable_get("@tubes_data"))
+    end
+  end # process_tube_options
+
   describe 'working a queue' do
     before do
       @worker = @worker_class.new(["foo:3"])
@@ -91,8 +107,13 @@ describe "Backburner::Workers::Threading worker" do
       assert_equal @thread_count, $worker_test_count
     end
 
+    # Its own tube and worker: the jobs it leaves reserved go back to ready when their connections are
+    # finalized, and must not reach another test's workers
     it 'forces an exit when a job is stuck' do
-      6.times { @worker_class.enqueue TestStuckJob, [1, 0], :queue => "threaded-shutdown" }
+      clear_jobs!("threaded-stuck")
+      @worker = @worker_class.new(["threaded-stuck:#{@thread_count}"])
+      @worker.exit_on_shutdown = false
+      6.times { @worker_class.enqueue TestStuckJob, [1, 0], :queue => "threaded-stuck" }
       Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
       capture_stdout do
         @worker.start