backburner-allq 1.0.0

data/test/test_helper.rb

@@ -0,0 +1,128 @@
+require 'rubygems'
+require 'tempfile'
+require 'minitest/autorun'
+begin
+  require 'mocha/setup'
+rescue LoadError
+  require 'mocha'
+end
+$LOAD_PATH.unshift File.expand_path("lib")
+require 'backburner'
+require File.expand_path('../helpers/templogger', __FILE__)
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.test"
+end
+
+## Kernel Extensions
+require 'stringio'
+
+module Kernel
+  # Redirect standard out, standard error and the buffered logger for sprinkle to StringIO
+  # capture_stdout { any_commands; you_want } => "all output from the commands"
+  def capture_stdout
+    if ENV['DEBUG'] # Skip if debug mode
+      yield
+      return ""
+    end
+
+    out = StringIO.new
+    $stdout = out
+    $stderr = out
+    yield
+    return out.string
+  ensure
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+end
+
+class User
+  attr_accessor :id, :name
+
+  def initialize(id, name)
+    @id, @name = id, name
+  end
+end
+
+class MiniTest::Spec
+  class << self
+    alias :should :it
+    alias :context :describe
+  end
+  alias :assert_no_match  :refute_match
+  alias :assert_not_nil   :refute_nil
+  alias :assert_not_equal :refute_equal
+
+  # assert_same_elements([:a, :b, :c], [:c, :a, :b]) => passes
+  def assert_same_elements(a1, a2, msg = nil)
+    [:select, :inject, :size].each do |m|
+      [a1, a2].each {|a| assert_respond_to(a, m, "Are you sure that #{a.inspect} is an array?  It doesn't respond to #{m}.") }
+    end
+
+    assert a1h = a1.inject({}) { |h,e| h[e] ||= a1.select { |i| i == e }.size; h }
+    assert a2h = a2.inject({}) { |h,e| h[e] ||= a2.select { |i| i == e }.size; h }
+
+    assert_equal(a1h, a2h, msg)
+  end
+
+  #   assert_contains(['a', '1'], /\d/) => passes
+  #   assert_contains(['a', '1'], 'a') => passes
+  #   assert_contains(['a', '1'], /not there/) => fails
+  def assert_contains(collection, x, extra_msg = "")
+    collection = [collection] unless collection.is_a?(Array)
+    msg = "#{x.inspect} not found in #{collection.to_a.inspect} #{extra_msg}"
+    case x
+    when Regexp
+      assert(collection.detect { |e| e =~ x }, msg)
+    else
+      assert(collection.include?(x), msg)
+    end
+  end
+
+  # silenced(5) { ... }
+  def silenced(time=3, &block)
+    Timeout::timeout(time) { capture_stdout(&block) }
+  end
+
+  def beanstalk_connection
+    Backburner::Connection.new(Backburner.configuration.beanstalk_url)
+  end
+
+  # pop_one_job(tube_name)
+  def pop_one_job(tube_name=Backburner.configuration.primary_queue)
+    tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
+    connection = beanstalk_connection
+    connection.tubes.watch!(tube_name)
+    silenced(3) { @res = connection.tubes.reserve }
+    yield @res, Backburner.configuration.job_parser_proc.call(@res.body)
+  ensure
+    connection.close if connection
+  end
+
+  # clear_jobs!('foo')
+  def clear_jobs!(*tube_names)
+    connection = beanstalk_connection
+    tube_names.each do |tube_name|
+      expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
+      connection.tubes.find(expanded_name).clear
+    end
+  ensure
+    connection.close if connection
+  end
+
+  # Simulates a broken connection for any Beaneater::Connection. Will
+  # simulate a restored connection after `reconnects_after`. This is expected
+  # to be used when ensuring a Beaneater connection is open, therefore
+  def simulate_disconnect(connection, reconnects_after = 2)
+    connection.beanstalk.connection.connection.expects(:closed? => true)
+    returns = Array.new(reconnects_after - 1, stub('TCPSocket'))
+    returns.each do |socket|
+      result = (socket != returns.last)
+      socket.stubs(:closed? => result)
+    end
+    TCPSocket.expects(:new).times(returns.size).returns(*returns)
+  end
+end # MiniTest::Spec

data/test/worker_test.rb

@@ -0,0 +1,157 @@
+require File.expand_path('../test_helper', __FILE__)
+require File.expand_path('../fixtures/test_jobs', __FILE__)
+require File.expand_path('../fixtures/hooked', __FILE__)
+
+describe "Backburner::Worker module" do
+  before do
+    Backburner.default_queues.clear
+    clear_jobs!(Backburner.configuration.primary_queue, "test-plain", "test.bar", "bar.baz.foo")
+  end
+
+  describe "for enqueue class method" do
+    it "should support enqueuing plain job" do
+      Backburner::Worker.enqueue TestPlainJob, [7, 9], :ttr => 100, :pri => 2000
+      pop_one_job("test-plain") do |job, body|
+        assert_equal "TestPlainJob", body["class"]
+        assert_equal [7, 9], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal 2000, job.pri
+      end
+    end # plain
+
+    it "should support enqueuing job with class queue priority" do
+      Backburner::Worker.enqueue TestJob, [3, 4], :ttr => 100
+      pop_one_job do |job, body|
+        assert_equal "TestJob", body["class"]
+        assert_equal [3, 4], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal 100, job.pri
+      end
+    end # queue priority
+
+    it "should support enqueuing job with specified named priority" do
+      Backburner::Worker.enqueue TestJob, [3, 4], :ttr => 100, :pri => 'high'
+      pop_one_job do |job, body|
+        assert_equal "TestJob", body["class"]
+        assert_equal [3, 4], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal 0, job.pri
+      end
+    end # queue named priority
+
+    it "should support enqueuing job with class queue respond_timeout" do
+      Backburner::Worker.enqueue TestJob, [3, 4]
+      pop_one_job do |job, body|
+        assert_equal "TestJob", body["class"]
+        assert_equal [3, 4], body["args"]
+        assert_equal 300, job.ttr
+        assert_equal 100, job.pri
+      end
+    end # queue respond_timeout
+
+    it "should support enqueuing job with custom queue" do
+      Backburner::Worker.enqueue TestJob, [6, 7], :queue => "test.bar", :pri => 5000
+      pop_one_job("test.bar") do |job, body|
+        assert_equal "TestJob", body["class"]
+        assert_equal [6, 7], body["args"]
+        assert_equal 0, job.delay
+        assert_equal 5000, job.pri
+        assert_equal 300, job.ttr
+      end
+    end # custom
+
+    it "should support async job" do
+      TestAsyncJob.async(:ttr => 100, :queue => "bar.baz.foo").foo(10, 5)
+      pop_one_job("bar.baz.foo") do |job, body|
+        assert_equal "TestAsyncJob", body["class"]
+        assert_equal [nil, "foo", 10, 5], body["args"]
+        assert_equal 100, job.ttr
+        assert_equal Backburner.configuration.default_priority, job.pri
+      end
+    end # async
+
+    it "should support enqueueing job with lambda queue" do
+      expected_queue_name = TestLambdaQueueJob.calculated_queue_name
+      Backburner::Worker.enqueue TestLambdaQueueJob, [6, 7], :queue => lambda { |klass| klass.calculated_queue_name }
+      pop_one_job(expected_queue_name) do |job, body|
+        assert_equal "TestLambdaQueueJob", body["class"]
+        assert_equal [6, 7], body["args"]
+      end
+    end
+  end # enqueue
+
+  describe "for start class method" do
+    it "should initialize and start the worker instance" do
+      ech = stub
+      Backburner::Worker.expects(:new).with("foo").returns(ech)
+      ech.expects(:start)
+      Backburner::Worker.start("foo")
+    end
+  end # start
+
+  describe "for tube_names accessor" do
+    before do
+      Backburner.default_queues << "baz"
+      Backburner.default_queues << "bam"
+    end
+
+    it "supports retrieving tubes" do
+      worker = Backburner::Worker.new(["foo", "bar"])
+      assert_equal ["foo", "bar"], worker.tube_names
+    end
+
+    it "supports single tube array arg" do
+      worker = Backburner::Worker.new([["foo", "bar"]])
+      assert_equal ["foo", "bar"], worker.tube_names
+    end
+
+    it "supports empty nil array arg with default values" do
+      worker = Backburner::Worker.new([nil])
+      assert_equal ['baz', 'bam'], worker.tube_names
+    end
+
+    it "supports single tube arg" do
+      worker = Backburner::Worker.new("foo")
+      assert_equal ["foo"], worker.tube_names
+    end
+
+    it "supports empty array arg with default values" do
+      worker = Backburner::Worker.new([])
+      assert_equal ['baz', 'bam'], worker.tube_names
+    end
+
+    it "supports nil arg with default values" do
+      worker = Backburner::Worker.new(nil)
+      assert_equal ['baz', 'bam'], worker.tube_names
+    end
+  end # tube_names
+
+  describe "for custom serialization" do
+    before do
+      Backburner.configure do |config|
+        @old_parser = config.job_parser_proc
+        @old_serializer = config.job_serializer_proc
+        config.job_parser_proc = lambda { |body| Marshal.load(body) }
+        config.job_serializer_proc = lambda { |body| Marshal.dump(body) }
+      end
+    end
+
+    after do
+      clear_jobs!('test-plain')
+      Backburner.configure do |config|
+        config.job_parser_proc = @old_parser
+        config.job_serializer_proc = @old_serializer
+      end
+    end
+
+    it "should support enqueuing a job" do
+      Backburner::Worker.enqueue TestPlainJob, [7, 9], :ttr => 100, :pri => 2000
+      pop_one_job("test-plain") do |job, body|
+        assert_equal "TestPlainJob", body[:class]
+        assert_equal [7, 9], body[:args]
+        assert_equal 100, job.ttr
+        assert_equal 2000, job.pri
+      end
+    end
+  end # custom serialization
+end # Backburner::Worker

data/test/workers/forking_worker_test.rb

@@ -0,0 +1,181 @@
+require File.expand_path('../../test_helper', __FILE__)
+require File.expand_path('../../fixtures/test_forking_jobs', __FILE__)
+
+describe "Backburner::Workers::Forking module" do
+
+  before do
+    Backburner.default_queues.clear
+    @worker_class = Backburner::Workers::Forking
+  end
+
+  describe "for prepare method" do
+    it "should make tube names array always unique to avoid duplication" do
+      worker = @worker_class.new(["foo", "demo.test.foo"])
+      capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo"], worker.tube_names
+    end
+
+    it "should watch specified tubes" do
+      worker = @worker_class.new(["foo", "bar"])
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.foo", "demo.test.bar"], worker.connection.tubes.watched.map(&:name)
+      assert_match(/demo\.test\.foo/, out)
+    end # multiple
+
+    it "should watch single tube" do
+      worker = @worker_class.new("foo")
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo"], worker.tube_names
+      assert_same_elements ["demo.test.foo"], worker.connection.tubes.watched.map(&:name)
+      assert_match(/demo\.test\.foo/, out)
+    end # single
+
+    it "should respect default_queues settings" do
+      Backburner.default_queues.concat(["foo", "bar"])
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.foo", "demo.test.bar"], worker.connection.tubes.watched.map(&:name)
+      assert_match(/demo\.test\.foo/, out)
+    end
+
+    it "should assign based on all tubes" do
+      @worker_class.any_instance.expects(:all_existing_queues).once.returns("bar")
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.bar"], worker.connection.tubes.watched.map(&:name)
+      assert_match(/demo\.test\.bar/, out)
+    end # all assign
+
+    it "should properly retrieve all tubes" do
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_contains worker.tube_names, "demo.test.backburner-jobs"
+      assert_contains worker.connection.tubes.watched.map(&:name), "demo.test.backburner-jobs"
+      assert_match(/demo\.test\.backburner-jobs/, out)
+    end # all read
+  end # prepare
+
+  describe "for fork_one_job method" do
+
+    it "should fork, work job, and exit" do
+      clear_jobs!("bar.foo")
+      @worker_class.enqueue TestJobForking, [1, 2], :queue => "bar.foo"
+
+      fake_pid = 45
+      Process.expects(:fork).returns(fake_pid).yields
+      @worker_class.any_instance.expects(:work_one_job)
+      @worker_class.any_instance.expects(:coolest_exit)
+      Process.expects(:wait).with(fake_pid)
+
+      silenced(2) do
+        worker = @worker_class.new('bar.foo')
+        worker.prepare
+        worker.fork_one_job
+      end
+    end
+
+  end # fork_one_job
+
+  describe "practical tests" do
+
+    before do
+      @templogger = Templogger.new('/tmp')
+      Backburner.configure { |config| config.logger = @templogger.logger }
+      $worker_test_count = 0
+      $worker_success = false
+      $worker_raise   = false
+      clear_jobs!('response')
+      clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
+      silenced do
+        @response_worker = @worker_class.new('response')
+      end
+    end
+
+    after do
+      @templogger.close
+      Backburner.configuration.logger = nil
+      clear_jobs!('response')
+      clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
+    end
+
+
+    it "should work an enqueued job" do
+      @worker = @worker_class.new('bar.foo.1')
+      @worker_class.enqueue TestJobForking, [1, 2], :queue => "bar.foo.1"
+      @worker.prepare
+      silenced(2) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Completed TestJobFork/m)
+        @response_worker.prepare
+        @response_worker.work_one_job
+      end
+      assert_equal 3, $worker_test_count
+    end # enqueue
+
+    it "should work for an async job" do
+      @worker = @worker_class.new('bar.foo.2')
+      TestAsyncJobForking.async(:queue => 'bar.foo.2').foo(3, 5)
+      @worker.prepare
+      silenced(4) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Completed TestAsyncJobFork/m)
+        @response_worker.prepare
+        @response_worker.work_one_job
+      end
+      assert_equal 15, $worker_test_count
+    end # async
+
+    it "should fail quietly if there's an argument error" do
+      Backburner.configure { |config| config.max_job_retries = 0 }
+      @worker = @worker_class.new('bar.foo.3')
+      @worker_class.enqueue TestJobForking, ["bam", "foo", "bar"], :queue => "bar.foo.3"
+      @worker.prepare
+      silenced(5) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Finished TestJobFork.*attempt 1 of 1/m)
+      end
+      assert_match(/Exception ArgumentError/, @templogger.body)
+      assert_equal 0, $worker_test_count
+    end # fail, argument
+
+    it "should support retrying jobs and burying" do
+      Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
+      @worker = @worker_class.new('bar.foo.4')
+      @worker_class.enqueue TestRetryJobForking, ["bam", "foo"], :queue => 'bar.foo.4'
+      @worker.prepare
+      silenced(4) do
+        2.times do
+          $worker_test_count += 1
+          @worker.fork_one_job
+        end
+        @templogger.wait_for_match(/Finished TestRetryJobFork.*attempt 2 of 2/m)
+        @response_worker.prepare
+        2.times { @response_worker.work_one_job }
+      end
+      assert_equal 4, $worker_test_count
+      assert_equal false, $worker_success
+    end # retry, bury
+
+    it "should support retrying jobs and succeeds" do
+      Backburner.configure { |config| config.max_job_retries = 2; config.retry_delay = 0 }
+      @worker = @worker_class.new('bar.foo.5')
+      @worker_class.enqueue TestRetryJobForking, ["bam", "foo"], :queue => 'bar.foo.5'
+      @worker.prepare
+      silenced(4) do
+        3.times do
+          $worker_test_count += 1
+          @worker.fork_one_job
+        end
+        @templogger.wait_for_match(/Completed TestRetryJobFork/m)
+        @response_worker.prepare
+        3.times { @response_worker.work_one_job }
+      end
+      assert_equal 6, $worker_test_count
+      assert_equal true, $worker_success
+    end # retrying, succeeds
+
+  end # practical tests
+end