backburner 1.1.0 → 1.2.0.pre

data/lib/backburner/workers/forking.rb

@@ -29,13 +29,17 @@ module Backburner
       # Waits for a job, works the job, and exits
       def fork_one_job
         pid = Process.fork do
-          @connection = Connection.new(Backburner.configuration.beanstalk_url)
           work_one_job
           coolest_exit
         end
         Process.wait(pid)
       end
 
+      def on_reconnect(conn)
+        @connection = conn
+        prepare
+      end
+
       # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
       # parent process
       # We will use exitcode 99 that means the fork reached the garbage number

data/lib/backburner/workers/simple.rb

@@ -26,4 +26,4 @@ module Backburner
       end
     end # Basic
   end # Workers
-end # Backburner
+end # Backburner

data/lib/backburner/workers/threads_on_fork.rb

@@ -50,6 +50,7 @@ module Backburner
 
         def finish_forks
           return if is_child
+
           ids = child_pids
           if ids.length > 0
             puts "[ThreadsOnFork workers] Stopping forks: #{ids.join(", ")}"
@@ -68,7 +69,7 @@ module Backburner
       # Custom initializer just to set @tubes_data
       def initialize(*args)
         @tubes_data = {}
-        @connection = nil
+        @mutex = Mutex.new
         super
         self.process_tube_options
       end
@@ -171,8 +172,6 @@ module Backburner
       # If we limit the number of threads to 1 it will just run in a loop without
       # creating any extra thread.
       def fork_inner(name)
-        watch_tube(name)
-
         if @tubes_data[name]
           queue_config.max_job_retries = @tubes_data[name][:retries] if @tubes_data[name][:retries]
         else
@@ -184,16 +183,18 @@ module Backburner
         @runs = 0
 
         if @threads_number == 1
-          run_while_can(name)
+          watch_tube(name)
+          run_while_can
         else
           threads_count = Thread.list.count
           @threads_number.times do
             create_thread do
-              conn = Connection.new(Backburner.configuration.beanstalk_url)
               begin
-                run_while_can(name, conn)
+                conn = new_connection
+                watch_tube(name, conn)
+                run_while_can(conn)
               ensure
-                conn.close
+                conn.close if conn
               end
             end
           end
@@ -204,21 +205,23 @@ module Backburner
       end
 
       # Run work_one_job while we can
-      def run_while_can(name, conn = nil)
-        conn ||= connection
-        watch_tube(name, conn)
+      def run_while_can(conn = connection)
         while @garbage_after.nil? or @garbage_after > @runs
-          @runs += 1
+          @runs += 1 # FIXME: Likely race condition
           work_one_job(conn)
         end
       end
 
-      # Shortcut for watching a tube on beanstalk connection
-      def watch_tube(name, conn = nil)
-        conn ||= connection
+      # Shortcut for watching a tube on our beanstalk connection
+      def watch_tube(name, conn = connection)
+        @watching_tube = name
         conn.tubes.watch!(name)
       end
 
+      def on_reconnect(conn)
+        watch_tube(@watching_tube, conn) if @watching_tube
+      end
+
       # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
       # parent process
       # We will use exitcode 99 that means the fork reached the garbage number
@@ -239,20 +242,17 @@ module Backburner
       end
 
       # Forks the specified block and adds the process to the child process pool
+      # FIXME: If blk.call breaks then the pid isn't added to child_pids and is
+      # never shutdown
       def fork_it(&blk)
         pid = Kernel.fork do
           self.class.is_child = true
           $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
-          @connection = Connection.new(Backburner.configuration.beanstalk_url)
           blk.call
         end
         self.class.child_pids << pid
         pid
       end
-
-      def connection
-        @connection || super
-      end
     end
   end
 end

data/test/async_proxy_test.rb

@@ -12,23 +12,25 @@ describe "Backburner::AsyncProxy class" do
     should "enqueue job onto worker with no args" do
       @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
       @async.foo
-      job, body = pop_one_job
-      assert_equal "AsyncUser", body["class"]
-      assert_equal [10, "foo"], body["args"]
-      assert_equal 100, job.ttr
-      assert_equal 1000, job.pri
-      job.delete
+      pop_one_job do |job, body|
+        assert_equal "AsyncUser", body["class"]
+        assert_equal [10, "foo"], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal 1000, job.pri
+        job.delete
+      end
     end
 
     should "enqueue job onto worker with args" do
       @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
       @async.bar(1, 2, 3)
-      job, body = pop_one_job
-      assert_equal "AsyncUser", body["class"]
-      assert_equal [10, "bar", 1, 2, 3], body["args"]
-      assert_equal 100, job.ttr
-      assert_equal 1000, job.pri
-      job.delete
+      pop_one_job do |job, body|
+        assert_equal "AsyncUser", body["class"]
+        assert_equal [10, "bar", 1, 2, 3], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal 1000, job.pri
+        job.delete
+      end
     end
   end # method_missing
-end # AsyncProxy
+end # AsyncProxy

data/test/connection_test.rb

@@ -3,11 +3,11 @@ require File.expand_path('../test_helper', __FILE__)
 describe "Backburner::Connection class" do
   describe "for initialize with single url" do
     before do
-      @connection = Backburner::Connection.new("beanstalk://localhost")
+      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
     end
 
     it "should store url in accessor" do
-      assert_equal "beanstalk://localhost", @connection.url
+      assert_equal "beanstalk://127.0.0.1", @connection.url
     end
 
     it "should setup beanstalk connection" do
@@ -17,9 +17,9 @@ describe "Backburner::Connection class" do
 
   describe "for initialize with url" do
     it "should delegate the address url correctly" do
-      @connection = Backburner::Connection.new("beanstalk://localhost")
+      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
       connection = @connection.beanstalk.connection
-      assert_equal 'localhost:11300', connection.address
+      assert_equal '127.0.0.1:11300', connection.address
     end
   end # initialize
 
@@ -31,13 +31,149 @@ describe "Backburner::Connection class" do
     end
   end
 
+  describe "for initialize with on_reconnect block" do
+    it "should store the block for use upon reconnect" do
+      callback = proc {}
+      connection = Backburner::Connection.new('beanstalk://127.0.0.1', &callback)
+      assert_equal callback, connection.on_reconnect
+    end
+  end
+
+  describe "dealing with connecting and reconnecting" do
+    before do
+      @connection = Backburner::Connection.new('beanstalk://127.0.0.1')
+    end
+
+    it "should know if its connection is open" do
+      assert_equal true, @connection.connected?
+      @connection.close
+      assert_equal false, @connection.connected?
+    end
+
+    it "should be able to attempt reconnecting to beanstalk" do
+      @connection.close
+      assert_equal false, @connection.connected?
+      @connection.reconnect!
+      assert_equal true, @connection.connected?
+    end
+
+    it "should allow for retryable commands" do
+      @result = false
+      @connection.close
+      @connection.retryable { @result = true }
+      assert_equal true, @result
+    end
+
+    it "should provide a hook when a retryable command successfully retries" do
+      @result = false
+      @retried = false
+      @connection.close
+      callback = proc { @result = true }
+      @connection.retryable(:on_retry => callback) do
+        unless @retried
+          @retried = true
+          raise Beaneater::NotConnected.new
+        end
+      end
+      assert_equal true, @result
+    end
+
+    it "should provide a hook when the connection successfully reconnects" do
+      reconnected = false
+      retried = false
+      @connection.close
+      @connection.on_reconnect = proc { reconnected = true }
+      @connection.retryable do
+        unless retried
+          retried = true
+          raise Beaneater::NotConnected.new
+        end
+      end
+      assert_equal true, reconnected
+    end
+
+    it "should call the on_reconnect hook before the on_retry hook" do
+      @result = []
+      @retried = false
+      @connection.close
+      @connection.on_reconnect = proc { @result << "reconnect" }
+      on_retry = proc { @result << "retry" }
+      @connection.retryable(:on_retry => on_retry) do
+        unless @retried
+          @retried = true
+          raise Beaneater::NotConnected.new
+        end
+      end
+      assert_equal %w(reconnect retry), @result
+    end
+
+    describe "ensuring the connection is open" do
+      it "should reattempt the connection to beanstalk several times" do
+        stats = @connection.stats
+        simulate_disconnect(@connection)
+        new_connection = Beaneater.new('127.0.0.1:11300')
+        Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(new_connection)
+        @connection.tubes
+        assert_equal true, @connection.connected?
+      end
+
+      it "should not attempt reconnecting if the current connection is open" do
+        assert_equal true, @connection.connected?
+        Beaneater.expects(:new).never
+        @connection.tubes
+      end
+
+      describe "when reconnecting is successful" do
+        it "should allow for a callback" do
+          @result = false
+          simulate_disconnect(@connection)
+          @connection.on_reconnect = proc { @result = true }
+          @connection.tubes
+          assert_equal true, @result
+        end
+
+        it "should pass self to the callback" do
+          result = nil
+          simulate_disconnect(@connection)
+          @connection.on_reconnect = lambda { |conn| result = conn }
+          @connection.tubes
+          assert_equal result, @connection
+        end
+      end
+    end
+
+    describe "when unable to ensure its connected" do
+      it "should raise Beaneater::NotConnected" do
+        Beaneater.stubs(:new).raises(Beaneater::NotConnected)
+        simulate_disconnect(@connection, 1) # since we're stubbing Beaneater.new above we only to simlulate the disconnect of our current connection
+        assert_raises Beaneater::NotConnected do
+          @connection.tubes
+        end
+      end
+    end
+
+    describe "when using the retryable method" do
+      it "should yield to the block multiple times" do
+        expected    = 2
+        retry_count = 0
+        @connection.retryable(max_retries: expected) do
+          if retry_count < 2
+            retry_count += 1
+            raise Beaneater::NotConnected
+          end
+        end
+        assert_equal expected, retry_count
+      end
+    end
+  end
+
   describe "for delegated methods" do
     before do
-      @connection = Backburner::Connection.new("beanstalk://localhost")
+      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
     end
 
     it "delegate methods to beanstalk connection" do
-      assert_equal "localhost", @connection.connection.host
+      assert_equal "127.0.0.1", @connection.connection.host
     end
   end # delegator
-end # Connection
+end # Connection

data/test/fixtures/hooked.rb

@@ -114,3 +114,9 @@ class HookedObjectSuccess
     puts "This is the job running successfully!! #{x.inspect}"
   end
 end # HookedObjectSuccess
+
+class HookedWorker < Backburner::Worker
+  def on_reconnect
+    puts "!!on_reconnect!!"
+  end
+end

data/test/fixtures/test_fork_jobs.rb

@@ -57,4 +57,16 @@ class TestAsyncJobFork
         :worker_test_count_set => x * y
     }], :queue => 'response'
   end
-end
+end
+
+class TestJobMultithreadFork
+  include Backburner::Queue
+  queue "test-job-multithread-fork"
+  queue_priority 1000
+  def self.perform(x, y)
+    sleep 1 # simluate work
+    Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+        :worker_test_count_set => x + y
+    }], :queue => 'response'
+  end
+end

data/test/helpers/templogger.rb

@@ -19,4 +19,4 @@ class Templogger
   def close
     @file.close
   end
-end
+end

data/test/hooks_test.rb

@@ -83,6 +83,13 @@ describe "Backburner::Hooks module" do
         assert_match(/!!on_bury_foo!! \[10\]/, out)
       end
     end
+
+    describe "with on_reconnect" do
+      it "should support successful invocation" do
+        out = silenced { @hooks.invoke_hook_events(HookedWorker.new, :on_reconnect)}
+        assert_match(/!!on_reconnect!!/, out)
+      end
+    end
   end # invoke_hook_events
 
   describe "for around_hook_events method" do

data/test/job_test.rb

@@ -111,4 +111,37 @@ describe "Backburner::Job module" do
       @job.bury
     end # bury
   end # simple delegation
+
+  describe "timing out for various values of ttr" do
+    before do
+      @task_body = { "class" => "NestedDemo::TestJobC", "args" => [56] }
+    end
+
+    describe "when ttr == 0" do
+      it "should use 0 for the timeout" do
+        @task = stub(:body => @task_body, :delete => true, :ttr => 0)
+        @job = Backburner::Job.new(@task)
+        Timeout.expects(:timeout).with(0)
+        @job.process
+      end
+    end
+
+    describe "when ttr == 1" do
+      it "should use 1 for the timeout" do
+        @task = stub(:body => @task_body, :delete => true, :ttr => 1)
+        @job = Backburner::Job.new(@task)
+        Timeout.expects(:timeout).with(1)
+        @job.process
+      end
+    end
+
+    describe "when ttr > 1" do
+      it "should use ttr-1 for the timeout" do
+        @task = stub(:body => @task_body, :delete => true, :ttr => 2)
+        @job = Backburner::Job.new(@task)
+        Timeout.expects(:timeout).with(1)
+        @job.process
+      end
+    end
+  end
 end

data/test/test_helper.rb

@@ -12,7 +12,7 @@ require File.expand_path('../helpers/templogger', __FILE__)
 
 # Configure Backburner
 Backburner.configure do |config|
-  config.beanstalk_url = "beanstalk://localhost"
+  config.beanstalk_url = "beanstalk://127.0.0.1"
   config.tube_namespace = "demo.test"
 end
 
@@ -25,7 +25,7 @@ module Kernel
   def capture_stdout
     if ENV['DEBUG'] # Skip if debug mode
       yield
-      ""
+      return ""
     end
 
     out = StringIO.new
@@ -87,20 +87,42 @@ class MiniTest::Spec
     Timeout::timeout(time) { capture_stdout(&block) }
   end
 
+  def beanstalk_connection
+    Backburner::Connection.new(Backburner.configuration.beanstalk_url)
+  end
+
   # pop_one_job(tube_name)
-  def pop_one_job(tube_name=Backburner.configuration.primary_queue)
-    connection = Backburner::Worker.connection
-    tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
+  def pop_one_job(tube_name=Backburner.configuration.primary_queue, &block)
+    tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
+    connection = beanstalk_connection
     connection.tubes.watch!(tube_name)
     silenced(3) { @res = connection.tubes.reserve }
-    return @res, JSON.parse(@res.body)
+    yield @res, JSON.parse(@res.body)
+  ensure
+    connection.close if connection
   end
 
   # clear_jobs!('foo')
   def clear_jobs!(*tube_names)
+    connection = beanstalk_connection
     tube_names.each do |tube_name|
       expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
-      Backburner::Worker.connection.tubes.find(expanded_name).clear
+      connection.tubes.find(expanded_name).clear
+    end
+  ensure
+    connection.close if connection
+  end
+
+  # Simulates a broken connection for any Beaneater::Connection. Will
+  # simulate a restored connection after `reconnects_after`. This is expected
+  # to be used when ensuring a Beaneater connection is open, therefore
+  def simulate_disconnect(connection, reconnects_after = 2)
+    connection.beanstalk.connection.connection.expects(:closed? => true)
+    returns = Array.new(reconnects_after - 1, stub('TCPSocket'))
+    returns.each do |socket|
+      result = (socket != returns.last)
+      socket.stubs(:closed? => result)
     end
+    TCPSocket.expects(:new).times(returns.size).returns(*returns)
   end
-end # MiniTest::Spec
+end # MiniTest::Spec