backburner 0.2.6 → 0.3.0

data/CHANGELOG.md

@@ -1,6 +1,12 @@
 # CHANGELOG
 
-## Version 0.2.7 (Unreleased)
+## Version 0.3.1 (Unreleased)
+
+## Version 0.3.0 (Nov 14 2012)
+
+ * Major update with support for a 'threads_on_fork' processing strategy (Thanks @ShadowBelmolve)
+ * Different workers have different rake tasks (Thanks @ShadowBelmolve)
+ * Added processing strategy specific examples i.e stress.rb and adds new unit tests. (Thanks @ShadowBelmolve)
 
 ## Version 0.2.6 (Nov 12 2012)
 

data/README.md

@@ -262,6 +262,7 @@ By default, Backburner comes with the following workers built-in:
 | Worker | Description                                                                 |
 | ------- | -------------------------------                                            |
 | `Backburner::Workers::Simple` | Single threaded, no forking worker. Simplest option. |
+| `Backburner::Workers::ThreadsOnFork` | Forking worker that utilizes threads for concurrent processing. |
 
 You can select the default worker for processing with:
 
@@ -274,12 +275,19 @@ end
 or determine the worker on the fly when invoking `work`:
 
 ```ruby
-Backburner.work('newsletter_sender', :worker => Backburner::Workers::Threaded)
+Backburner.work('newsletter_sender', :worker => Backburner::Workers::ThreadsOnFork)
 ```
 
-or when more official workers are supported, through alternate rake tasks.
-Additional workers such as `threaded`, `forking` and `threads_on_fork` will hopefully be
-developed in the future. If you are interested in helping, please let us know.
+or through associated rake tasks with:
+
+```
+$ QUEUES=newsletter-sender,push-message THREADS=2 GARBAGE=1000 rake backburner:threads_on_fork:work
+```
+
+For more information on the threads_on_fork worker, check out the
+[ThreadsOnFork Worker](https://github.com/nesquena/backburner/wiki/ThreadsOnFork-worker) documentation.
+Additional workers such as individual `threaded` and `forking` strategies will hopefully be contributed in the future.
+If you are interested in helping out, please let us know.
 
 ### Default Queues
 
@@ -391,6 +399,7 @@ jobs processed by your beanstalk workers. An excellent addition to your Backburn
  * Kristen Tucker - Coming up with the gem name
  * [Tim Lee](https://github.com/timothy1ee), [Josh Hull](https://github.com/joshbuddy), [Nico Taing](https://github.com/Nico-Taing) - Helping me work through the idea
  * [Miso](http://gomiso.com) - Open-source friendly place to work
+ * [Renan T. Fernandes](https://github.com/ShadowBelmolve) - Added threads_on_fork worker
 
 ## Contributing
 

data/backburner.gemspec

@@ -19,6 +19,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'dante', '~> 0.1.5'
 
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'minitest', '~> 4.1.0'
+  s.add_development_dependency 'minitest', '3.2.0'
   s.add_development_dependency 'mocha'
 end

data/examples/custom.rb

@@ -21,5 +21,5 @@ end
 Backburner.enqueue TestJob, 5, 3
 Backburner.enqueue TestJob, 10, 6
 
-# Work tasks
-Backburner.work("test-job")
+# Work tasks using threaded worker
+Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)

data/examples/stress.rb

@@ -0,0 +1,30 @@
+$:.unshift "lib"
+require 'backburner'
+
+$values = []
+
+# Define ruby job
+class TestJob
+  include Backburner::Queue
+  queue "test-job"
+
+  def self.perform(value)
+    puts "[TestJob] Running perform with args: [#{value}]"
+    $values << value
+    puts "#{$values.size} total jobs processed"
+  end
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.tube_namespace = "demo.production"
+end
+
+# Enqueue tasks
+1.upto(1000) do |i|
+  Backburner.enqueue TestJob, i
+end
+
+# Work tasks using threaded worker
+Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)

data/lib/backburner.rb

@@ -11,6 +11,7 @@ require 'backburner/hooks'
 require 'backburner/performable'
 require 'backburner/worker'
 require 'backburner/workers/simple'
+require 'backburner/workers/threads_on_fork'
 require 'backburner/queue'
 
 module Backburner

data/lib/backburner/async_proxy.rb

@@ -11,7 +11,7 @@ module Backburner
     # Options include `pri` (priority), `delay` (delay in secs), `ttr` (time to respond)
     #
     # @example
-    #   AsyncProxy(User, 10, :pri => 1000, :ttr => 1000)
+    #   AsyncProxy.new(User, 10, :pri => 1000, :ttr => 1000)
     #
     def initialize(klazz, id=nil, opts={})
       @klazz, @id, @opts = klazz, id, opts

data/lib/backburner/logger.rb

@@ -14,10 +14,12 @@ module Backburner
     end
 
     # Print out when a job completed
+    # If message is nil, job is considered complete
     def log_job_end(name, message = nil)
       ellapsed = Time.now - job_started_at
       ms = (ellapsed.to_f * 1000).to_i
-      log_info("Finished #{name} in #{ms}ms #{message}")
+      action_word = message ? 'Finished' : 'Completed'
+      log_info("#{action_word} #{name} in #{ms}ms #{message}")
     end
 
     # Returns true if the job logging started

data/lib/backburner/tasks.rb

@@ -3,9 +3,32 @@
 
 namespace :backburner do
   # QUEUE=foo,bar,baz rake backburner:work
-  desc "Start an backburner worker"
+  desc "Start backburner worker using default worker"
   task :work => :environment do
     queues = (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
     Backburner.work queues
   end
+
+  namespace :simple do
+    # QUEUE=foo,bar,baz rake backburner:simple:work
+    desc "Starts backburner worker using simple processing"
+    task :work => :environment do
+      queues = (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
+      Backburner.work queues, :worker => Backburner::Workers::Simple
+    end
+  end # simple
+
+  namespace :threads_on_fork do
+    # QUEUE=twitter:10:5000:5,parse_page,send_mail,verify_bithday THREADS=2 GARBAGE=1000 rake backburner:threads_on_fork:work
+    # twitter tube will have 10 threads, garbage after 5k executions and retry 5 times.
+    desc "Starts backburner worker using threads_on_fork processing"
+    task :work => :environment do
+      queues = (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
+      threads = ENV['THREADS'].to_i
+      garbage = ENV['GARBAGE'].to_i
+      Backburner::Workers::ThreadsOnFork.threads_number = threads if threads > 0
+      Backburner::Workers::ThreadsOnFork.garbage_after  = garbage if garbage > 0
+      Backburner.work queues, :worker => Backburner::Workers::ThreadsOnFork
+    end
+  end # threads_on_fork
 end

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/lib/backburner/workers/threads_on_fork.rb

@@ -0,0 +1,239 @@
+module Backburner
+  module Workers
+    class ThreadsOnFork < Worker
+
+      class << self
+        attr_accessor :shutdown
+        attr_accessor :threads_number
+        attr_accessor :garbage_after
+        attr_accessor :is_child
+
+        # return the pids of all alive children/forks
+        def child_pids
+          return [] if is_child
+          @child_pids ||= []
+          tmp_ids = []
+          for id in @child_pids
+            next if id.to_i == Process.pid
+            begin
+              Process.kill(0, id)
+              tmp_ids << id
+            rescue Errno::ESRCH => e
+            end
+          end
+          @child_pids = tmp_ids if @child_pids != tmp_ids
+          @child_pids
+        end
+
+        # Send a SIGTERM signal to all children
+        # This is the same of a normal exit
+        # We are simply asking the children to exit
+        def stop_forks
+          for id in child_pids
+            begin
+              Process.kill("SIGTERM", id)
+            rescue Errno::ESRCH
+            end
+          end
+        end
+
+        # Send a SIGKILL signal to all children
+        # This is the same of assassinate
+        # We are KILLING those folks that don't obey us
+        def kill_forks
+          for id in child_pids
+            begin
+              Process.kill("SIGKILL", id)
+            rescue Errno::ESRCH
+            end
+          end
+        end
+
+        def finish_forks
+          return if is_child
+          ids = child_pids
+          if ids.length > 0
+            puts "[ThreadsOnFork workers] Stopping forks: #{ids.join(", ")}"
+            stop_forks
+            Kernel.sleep 1
+            ids = child_pids
+            if ids.length > 0
+              puts "[ThreadsOnFork workers] Killing remaining forks: #{ids.join(", ")}"
+              kill_forks
+              Process.waitall
+            end
+          end
+        end
+      end
+
+      # Custom initializer just to set @tubes_data
+      def initialize(*args)
+        @tubes_data = {}
+        super
+      end
+
+      # Process the special tube_names of ThreadsOnFork worker
+      # The idea is tube_name:custom_threads_limit:custom_garbage_limit:custom_retries
+      # Any custom can be ignore. So if you want to set just the custom_retries
+      # you will need to write this 'tube_name:::10'
+      #
+      # @example
+      #    process_tube_names(['foo:10:5:1', 'bar:2::3', 'lol'])
+      #    => ['foo', 'bar', 'lol']
+      def process_tube_names(tube_names)
+        names = compact_tube_names(tube_names)
+        if names.nil?
+          nil
+        else
+          names.map do |name|
+            data = name.split(":")
+            tube_name = data.first
+            threads_number = data[1].empty? ? nil : data[1].to_i rescue nil
+            garbage_number = data[2].empty? ? nil : data[2].to_i rescue nil
+            retries_number = data[3].empty? ? nil : data[3].to_i rescue nil
+            @tubes_data[expand_tube_name(tube_name)] = {
+                :threads => threads_number,
+                :garbage => garbage_number,
+                :retries => retries_number
+            }
+            tube_name
+          end
+        end
+      end
+
+      def prepare
+        self.tube_names ||= Backburner.default_queues.any? ? Backburner.default_queues : all_existing_queues
+        self.tube_names = Array(self.tube_names)
+        tube_names.map! { |name| expand_tube_name(name)  }
+        log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+      end
+
+      # For each tube we will call fork_and_watch to create the fork
+      # The lock argument define if this method should block or no
+      def start(lock=true)
+        prepare
+        tube_names.each do |name|
+          fork_and_watch(name)
+        end
+
+        if lock
+          sleep 0.1 while true
+        end
+      end
+
+      # Make the fork and create a thread to watch the child process
+      # The exit code '99' means that the fork exited because of the garbage limit
+      # Any other code is an error
+      def fork_and_watch(name)
+        create_thread(name) do |tube_name|
+          until self.class.shutdown
+            pid = fork_tube(tube_name)
+            _, status = wait_for_process(pid)
+
+            # 99 = garbaged
+            if status.exitstatus != 99
+              log_error("Catastrophic failure: tube #{tube_name} exited with code #{status.exitstatus}.")
+            end
+          end
+        end
+      end
+
+      # This makes easy to test
+      def fork_tube(name)
+        fork_it do
+          fork_inner(name)
+        end
+      end
+
+      # Here we are already on the forked child
+      # We will watch just the selected tube and change the configuration of
+      # config.max_job_retries if needed
+      #
+      # If we limit the number of threads to 1 it will just run in a loop without
+      # creating any extra thread.
+      def fork_inner(name)
+        watch_tube(name)
+
+        if @tubes_data[name]
+          config.max_job_retries = @tubes_data[name][:retries] if @tubes_data[name][:retries]
+        else
+          @tubes_data[name] = {}
+        end
+        @garbage_after  = @tubes_data[name][:garbage]  || self.class.garbage_after
+        @threads_number = (@tubes_data[name][:threads] || self.class.threads_number || 1).to_i
+
+        @runs = 0
+
+        if @threads_number == 1
+          run_while_can
+        else
+          threads_count = Thread.list.count
+          @threads_number.times do
+            create_thread do
+              run_while_can
+            end
+          end
+          sleep 0.1 while Thread.list.count > threads_count
+        end
+
+        coolest_exit
+      end
+
+      # Run work_one_job while we can
+      def run_while_can
+        while @garbage_after.nil? or @garbage_after > @runs
+          @runs += 1
+          work_one_job
+        end
+      end
+
+      # Shortcut for watching a tube on beanstalk connection
+      def watch_tube(name)
+        connection.tubes.watch!(name)
+      end
+
+      # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
+      # parent process
+      # We will use exitcode 99 that means the fork reached the garbage number
+      def coolest_exit
+        Kernel.exit! 99
+      end
+
+      # Create a thread. Easy to test
+      def create_thread(*args, &block)
+        Thread.new(*args, &block)
+      end
+
+      # Wait for a specific process. Easy to test
+      def wait_for_process(pid)
+        out = Process.wait2(pid)
+        self.class.child_pids.delete(pid)
+        out
+      end
+
+      # Forks the specified block and adds the process to the child process pool
+      def fork_it(&blk)
+        pid = Kernel.fork do
+          self.class.is_child = true
+          $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
+          @connection = Connection.new(Backburner.configuration.beanstalk_url)
+          blk.call
+        end
+        self.class.child_pids << pid
+        pid
+      end
+
+      def connection
+        @connection || super
+      end
+
+    end
+  end
+end
+
+at_exit do
+  unless Backburner::Workers::ThreadsOnFork.is_child
+    Backburner::Workers::ThreadsOnFork.shutdown = true
+  end
+  Backburner::Workers::ThreadsOnFork.finish_forks
+end

data/test/async_proxy_test.rb

@@ -0,0 +1,37 @@
+require File.expand_path('../test_helper', __FILE__)
+
+class AsyncUser; def self.invoke_hook_events(*args); true; end; end
+
+describe "Backburner::AsyncProxy class" do
+  before do
+    Backburner.default_queues.clear
+  end
+
+  after do
+    clear_jobs!("async-user")
+  end
+
+  describe "for method_missing enqueue" do
+    should "enqueue job onto worker with no args" do
+      @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
+      @async.foo
+      job, body = pop_one_job("async-user")
+      assert_equal "AsyncUser", body["class"]
+      assert_equal [10, "foo"], body["args"]
+      assert_equal 100, job.ttr
+      assert_equal 1000, job.pri
+      job.delete
+    end
+
+    should "enqueue job onto worker with args" do
+      @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
+      @async.bar(1, 2, 3)
+      job, body = pop_one_job("async-user")
+      assert_equal "AsyncUser", body["class"]
+      assert_equal [10, "bar", 1, 2, 3], body["args"]
+      assert_equal 100, job.ttr
+      assert_equal 1000, job.pri
+      job.delete
+    end
+  end # method_missing
+end # AsyncProxy

data/test/fixtures/test_fork_jobs.rb

@@ -0,0 +1,58 @@
+class ResponseJob
+  include Backburner::Queue
+  queue_priority 1000
+  def self.perform(data)
+    $worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
+    $worker_success = data['worker_success'] if data['worker_success']
+    $worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
+    $worker_raise = data['worker_raise'] if data['worker_raise']
+  end
+end
+
+class TestJobFork
+  include Backburner::Queue
+  queue_priority 1000
+  def self.perform(x, y)
+    Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+        :worker_test_count_set => x + y
+    }], :queue => 'response'
+  end
+end
+
+class TestFailJobFork
+  include Backburner::Queue
+  def self.perform(x, y)
+    Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+       :worker_raise => true
+    }], :queue => 'response'
+  end
+end
+
+class TestRetryJobFork
+  include Backburner::Queue
+  def self.perform(x, y)
+    $worker_test_count += 1
+
+    if $worker_test_count <= 2
+      Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+          :worker_test_count => 1
+      }], :queue => 'response'
+
+      raise RuntimeError
+    else # succeeds
+      Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+          :worker_test_count => 1,
+          :worker_success => true
+      }], :queue => 'response'
+    end
+  end
+end
+
+class TestAsyncJobFork
+  include Backburner::Performable
+  def self.foo(x, y)
+    Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
+        :worker_test_count_set => x * y
+    }], :queue => 'response'
+  end
+end

data/test/helpers/templogger.rb

@@ -0,0 +1,22 @@
+class Templogger
+  attr_reader :logger, :log_path
+
+  def initialize(root_path)
+    @file = Tempfile.new('foo', root_path)
+    @log_path = @file.path
+    @logger = Logger.new(@log_path)
+  end
+
+  # wait_for_match /Completed TestJobFork/m
+  def wait_for_match(match_pattern)
+    sleep 0.1 until self.body =~ match_pattern
+  end
+
+  def body
+    File.read(@log_path)
+  end
+
+  def close
+    @file.close
+  end
+end

data/test/test_helper.rb

@@ -4,6 +4,7 @@ require 'minitest/autorun'
 require 'mocha'
 $:.unshift File.expand_path("../../lib")
 require 'backburner'
+require File.expand_path('../helpers/templogger', __FILE__)
 
 # Configure Backburner
 Backburner.configure do |config|

data/test/{simple_worker_test.rb → workers/simple_worker_test.rb}

@@ -1,6 +1,6 @@
-require File.expand_path('../test_helper', __FILE__)
-require File.expand_path('../fixtures/test_jobs', __FILE__)
-require File.expand_path('../fixtures/hooked', __FILE__)
+require File.expand_path('../../test_helper', __FILE__)
+require File.expand_path('../../fixtures/test_jobs', __FILE__)
+require File.expand_path('../../fixtures/hooked', __FILE__)
 
 describe "Backburner::Workers::Basic module" do
   before do
@@ -149,7 +149,7 @@ describe "Backburner::Workers::Basic module" do
       end
       assert_match /attempt 1 of 3, retrying/, out.first
       assert_match /attempt 2 of 3, retrying/, out[1]
-      assert_match /Finished TestRetryJob/m, out.last
+      assert_match /Completed TestRetryJob/m, out.last
       refute_match(/failed/, out.last)
       assert_equal 3, $worker_test_count
       assert_equal true, $worker_success
@@ -223,7 +223,7 @@ describe "Backburner::Workers::Basic module" do
         worker.work_one_job
       end
       assert_match /!!before_perform_foo!! \[nil, "foo", 10\]/, out
-      assert_match /before_perform_foo.*Finished/m, out
+      assert_match /before_perform_foo.*Completed/m, out
       refute_match(/Fail ran!!/, out)
       refute_match(/HookFailError/, out)
     end # stopping perform

data/test/workers/threads_on_fork_test.rb

@@ -0,0 +1,412 @@
+require File.expand_path('../../test_helper', __FILE__)
+require File.expand_path('../../fixtures/test_fork_jobs', __FILE__)
+
+describe "Backburner::Workers::ThreadsOnFork module" do
+
+  before do
+    Backburner.default_queues.clear
+    @worker_class = Backburner::Workers::ThreadsOnFork
+    @worker_class.shutdown = false
+    @worker_class.is_child = false
+    @worker_class.threads_number = 1
+    @worker_class.garbage_after = 1
+    @ignore_forks = false
+  end
+
+  after do
+    Backburner.configure { |config| config.max_job_retries = 0; config.retry_delay = 5; config.logger = nil }
+    unless @ignore_forks
+      if @worker_class.instance_variable_get("@child_pids").length > 0
+        raise "Why is there forks alive?"
+      end
+    end
+  end
+
+  describe "for process_tube_names method" do
+    it "should interpreter the job_name:threads_limit:garbage_after:retries format" do
+      worker = @worker_class.new(["foo:1:2:3"])
+      assert_equal ["foo"], worker.tube_names
+    end
+
+    it "should interpreter event if is missing values" do
+      tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
+      worker = @worker_class.new(tubes)
+      assert_equal %W(foo1 foo2 foo3 foo4 foo5 foo6 foo7), worker.tube_names
+    end
+
+    it "should store interpreted values correctly" do
+      tubes = %W(foo1:1:2:3 foo2:4:5 foo3:6 foo4 foo5::7:8 foo6:::9 foo7::10)
+      worker = @worker_class.new(tubes)
+      assert_equal({
+        "demo.test.foo1" => { :threads => 1,   :garbage => 2,   :retries => 3   },
+        "demo.test.foo2" => { :threads => 4,   :garbage => 5,   :retries => nil },
+        "demo.test.foo3" => { :threads => 6,   :garbage => nil, :retries => nil },
+        "demo.test.foo4" => { :threads => nil, :garbage => nil, :retries => nil },
+        "demo.test.foo5" => { :threads => nil, :garbage => 7,   :retries => 8   },
+        "demo.test.foo6" => { :threads => nil, :garbage => nil, :retries => 9   },
+        "demo.test.foo7" => { :threads => nil, :garbage => 10,  :retries => nil }
+      }, worker.instance_variable_get("@tubes_data"))
+    end
+  end
+
+  describe "for prepare method" do
+    it "should watch specified tubes" do
+      worker = @worker_class.new(["foo", "bar"])
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_match /demo\.test\.foo/, out
+    end # multiple
+
+    it "should watch single tube" do
+      worker = @worker_class.new("foo")
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo"], worker.tube_names
+      assert_match /demo\.test\.foo/, out
+    end # single
+
+    it "should respect default_queues settings" do
+      Backburner.default_queues.concat(["foo", "bar"])
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_match /demo\.test\.foo/, out
+    end
+
+    it "should assign based on all tubes" do
+      @worker_class.any_instance.expects(:all_existing_queues).once.returns("bar")
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.bar"], worker.tube_names
+      assert_match /demo\.test\.bar/, out
+    end # all assign
+
+    it "should properly retrieve all tubes" do
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_contains worker.tube_names, "demo.test.test-job-fork"
+      assert_match /demo\.test\.test-job-fork/, out
+    end # all read
+  end # prepare
+
+  describe "forking and threading" do
+
+    it "start should call fork_and_watch for each tube" do
+      worker = @worker_class.new(%W(foo bar))
+      worker.expects(:fork_and_watch).with("demo.test.foo").once
+      worker.expects(:fork_and_watch).with("demo.test.bar").once
+      silenced { worker.start(false) }
+    end
+
+    it "fork_and_watch should create a thread to fork and watch" do
+      worker = @worker_class.new(%(foo))
+      worker.expects(:create_thread).once.with("demo.test.foo")
+      silenced { worker.start(false) }
+    end
+
+    it "fork_and_watch thread should wait with wait_for_process" do
+      process_exit = stub('process_exit')
+      process_exit.expects(:exitstatus).returns(99)
+      worker = @worker_class.new(%(foo))
+      worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
+
+      wc = @worker_class
+      # TODO: Is there a best way do do this?
+      worker.define_singleton_method :fork_it do
+        wc.shutdown = true
+        12
+      end
+      def worker.create_thread(*args, &block); block.call(*args) end
+
+      out = silenced(2) { worker.start(false) }
+      refute_match /Catastrophic failure/, out
+    end
+
+    it "fork_and_watch thread should log an error if exitstatus is != 99" do
+      process_exit = stub('process_exit')
+      process_exit.expects(:exitstatus).twice.returns(0)
+      worker = @worker_class.new(%(foo))
+      worker.expects(:wait_for_process).with(12).returns([nil, process_exit])
+
+      wc = @worker_class
+      # TODO: Is there a best way do do this?
+      worker.define_singleton_method :fork_it do
+        wc.shutdown = true
+        12
+      end
+      def worker.create_thread(*args, &block); block.call(*args) end
+      out = silenced(2) { worker.start(false) }
+      assert_match /Catastrophic failure: tube demo\.test\.foo exited with code 0\./, out
+    end
+
+    describe "fork_inner" do
+
+      before do
+        @worker_class.any_instance.expects(:coolest_exit).once
+      end
+
+      it "should watch just the channel it receive as argument" do
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(1)
+        worker.expects(:run_while_can).once
+        silenced do
+          worker.prepare
+          worker.fork_inner('demo.test.bar')
+        end
+        assert_same_elements %W(demo.test.bar), @worker_class.connection.tubes.watched.map(&:name)
+      end
+
+      it "should not create threads if the number of threads is 1" do
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(1)
+        worker.expects(:run_while_can).once
+        worker.expects(:create_thread).never
+        silenced do
+          worker.prepare
+          worker.fork_inner('demo.test.foo')
+        end
+      end
+
+      it "should create threads if the number of threads is > 1" do
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(5)
+        worker.expects(:create_thread).times(5)
+        silenced do
+          worker.prepare
+          worker.fork_inner('demo.test.foo')
+        end
+      end
+
+      it "should create threads that call run_while_can" do
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(5)
+        worker.expects(:run_while_can).times(5)
+        # TODO
+        def worker.create_thread(*args, &block); block.call(*args) end
+        silenced do
+          worker.prepare
+          worker.fork_inner('demo.test.foo')
+        end
+      end
+
+      it "should set @garbage_after, @threads_number and set retries if needed" do
+        worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
+        default_threads = 1
+        default_garbage = 5
+        default_retries = 100
+        @worker_class.expects(:threads_number).times(1).returns(default_threads)
+        @worker_class.expects(:garbage_after).times(2).returns(default_garbage)
+        @worker_class.any_instance.expects(:coolest_exit).times(3)
+        Backburner.configuration.max_job_retries = default_retries
+
+        worker.expects(:create_thread).times(70)
+        worker.expects(:run_while_can).once
+
+        silenced do
+          worker.prepare
+          worker.fork_inner('demo.test.foo1')
+        end
+
+        assert_equal worker.instance_variable_get("@threads_number"), default_threads
+        assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
+        assert_equal Backburner.configuration.max_job_retries, default_retries
+
+        silenced do
+          worker.fork_inner('demo.test.foo2')
+        end
+
+        assert_equal worker.instance_variable_get("@threads_number"), 10
+        assert_equal worker.instance_variable_get("@garbage_after"), default_garbage
+        assert_equal Backburner.configuration.max_job_retries, default_retries
+
+        silenced do
+          worker.fork_inner('demo.test.foo3')
+        end
+
+        assert_equal worker.instance_variable_get("@threads_number"), 20
+        assert_equal worker.instance_variable_get("@garbage_after"), 30
+        assert_equal Backburner.configuration.max_job_retries, default_retries
+
+        silenced do
+          worker.fork_inner('demo.test.foo4')
+        end
+
+        assert_equal worker.instance_variable_get("@threads_number"), 40
+        assert_equal worker.instance_variable_get("@garbage_after"), 50
+        assert_equal Backburner.configuration.max_job_retries, 60
+      end
+
+    end
+
+    describe "cleanup on parent" do
+
+      it "child_pids should return a list of alive children pids" do
+        worker = @worker_class.new(%W(foo))
+        Kernel.expects(:fork).once.returns(12345)
+        Process.expects(:kill).with(0, 12345).once
+        Process.expects(:pid).once.returns(12346)
+        assert_equal [], @worker_class.child_pids
+        worker.fork_it {}
+        child_pids = @worker_class.child_pids
+        assert_equal [12345], child_pids
+        child_pids.clear
+      end
+
+      it "child_pids should return an empty array if is_child" do
+        Process.expects(:pid).never
+        @worker_class.is_child = true
+        @worker_class.child_pids << 12345
+        assert_equal [], @worker_class.child_pids
+      end
+
+      it "stop_forks should send a SIGTERM for every child" do
+        Process.expects(:pid).returns(12346).at_least(1)
+        Process.expects(:kill).with(0, 12345).at_least(1)
+        Process.expects(:kill).with(0, 12347).at_least(1)
+        Process.expects(:kill).with("SIGTERM", 12345)
+        Process.expects(:kill).with("SIGTERM", 12347)
+        @worker_class.child_pids << 12345
+        @worker_class.child_pids << 12347
+        assert_equal [12345, 12347], @worker_class.child_pids
+        @worker_class.stop_forks
+        @worker_class.child_pids.clear
+      end
+
+      it "kill_forks should send a SIGKILL for every child" do
+        Process.expects(:pid).returns(12346).at_least(1)
+        Process.expects(:kill).with(0, 12345).at_least(1)
+        Process.expects(:kill).with(0, 12347).at_least(1)
+        Process.expects(:kill).with("SIGKILL", 12345)
+        Process.expects(:kill).with("SIGKILL", 12347)
+        @worker_class.child_pids << 12345
+        @worker_class.child_pids << 12347
+        assert_equal [12345, 12347], @worker_class.child_pids
+        @worker_class.kill_forks
+        @worker_class.child_pids.clear
+      end
+
+      it "finish_forks should call stop_forks, kill_forks and Process.waitall" do
+        Process.expects(:pid).returns(12346).at_least(1)
+        Process.expects(:kill).with(0, 12345).at_least(1)
+        Process.expects(:kill).with(0, 12347).at_least(1)
+        Process.expects(:kill).with("SIGTERM", 12345)
+        Process.expects(:kill).with("SIGTERM", 12347)
+        Process.expects(:kill).with("SIGKILL", 12345)
+        Process.expects(:kill).with("SIGKILL", 12347)
+        Kernel.expects(:sleep).with(1)
+        Process.expects(:waitall)
+        @worker_class.child_pids << 12345
+        @worker_class.child_pids << 12347
+        assert_equal [12345, 12347], @worker_class.child_pids
+        silenced do
+          @worker_class.finish_forks
+        end
+        @worker_class.child_pids.clear
+      end
+
+      it "finish_forks should not do anything if is_child" do
+        @worker_class.expects(:stop_forks).never
+        @worker_class.is_child = true
+        @worker_class.child_pids << 12345
+        silenced do
+          @worker_class.finish_forks
+        end
+      end
+
+    end # cleanup on parent
+
+    describe "practical tests" do
+
+      before do
+        @templogger = Templogger.new('/tmp')
+        Backburner.configure { |config| config.logger = @templogger.logger }
+        $worker_test_count = 0
+        $worker_success = false
+        $worker_raise   = false
+        clear_jobs!('response')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5')
+        @worker_class.threads_number = 1
+        @worker_class.garbage_after  = 10
+        silenced do
+          @response_worker = @worker_class.new('response')
+          @response_worker.watch_tube('demo.test.response')
+        end
+        @ignore_forks = true
+      end
+
+      after do
+        @templogger.close
+        clear_jobs!('response')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5')
+        @worker_class.shutdown = true
+        silenced do
+          @worker_class.stop_forks
+          Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
+          @worker_class.kill_forks
+          Timeout::timeout(2) { sleep 0.1 while @worker_class.child_pids.length > 0 }
+        end
+      end
+
+      it "should work an enqueued job" do
+        @worker = @worker_class.new('foo.bar.1')
+        @worker.start(false)
+        @worker_class.enqueue TestJobFork, [1, 2], :queue => "foo.bar.1"
+        silenced(2) do
+          @templogger.wait_for_match(/Completed TestJobFork/m)
+          @response_worker.work_one_job
+        end
+        assert_equal 3, $worker_test_count
+      end # enqueue
+
+      it "should work for an async job" do
+        @worker = @worker_class.new('foo.bar.2')
+        @worker.start(false)
+        TestAsyncJobFork.async(:queue => 'foo.bar.2').foo(3, 5)
+        silenced(2) do
+          @templogger.wait_for_match(/Completed TestAsyncJobFork/m)
+          @response_worker.work_one_job
+        end
+        assert_equal 15, $worker_test_count
+      end # async
+
+      it "should fail quietly if there's an argument error" do
+        @worker = @worker_class.new('foo.bar.3')
+        @worker.start(false)
+        @worker_class.enqueue TestJobFork, ["bam", "foo", "bar"], :queue => "foo.bar.3"
+        silenced(5) do
+          @templogger.wait_for_match(/Finished TestJobFork.*attempt 1 of 1/m)
+        end
+        assert_match(/Exception ArgumentError/, @templogger.body)
+        assert_equal 0, $worker_test_count
+      end # fail, argument
+
+      it "should support retrying jobs and burying" do
+        Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
+        @worker = @worker_class.new('foo.bar.4')
+        @worker.start(false)
+        @worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.4'
+        silenced(2) do
+          @templogger.wait_for_match(/Finished TestRetryJobFork.*attempt 2 of 2/m)
+          2.times { @response_worker.work_one_job }
+        end
+        assert_equal 2, $worker_test_count
+        assert_equal false, $worker_success
+      end # retry, bury
+
+      it "should support retrying jobs and succeeds" do
+        Backburner.configure { |config| config.max_job_retries = 2; config.retry_delay = 0 }
+        @worker = @worker_class.new('foo.bar.5')
+        @worker.start(false)
+        @worker_class.enqueue TestRetryJobFork, ["bam", "foo"], :queue => 'foo.bar.5'
+        silenced(2) do
+          @templogger.wait_for_match(/Completed TestRetryJobFork/m)
+          3.times { @response_worker.work_one_job }
+        end
+        assert_equal 3, $worker_test_count
+        assert_equal true, $worker_success
+      end # retrying, succeeds
+
+    end # practical tests
+
+  end # forking and threading
+
+end # Backburner::Workers::ThreadsOnFork module

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-13 00:00:00.000000000 Z
+date: 2012-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -64,17 +64,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 3.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.0
+        version: 3.2.0
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +117,7 @@ files:
 - examples/hooked.rb
 - examples/retried.rb
 - examples/simple.rb
+- examples/stress.rb
 - lib/backburner.rb
 - lib/backburner/async_proxy.rb
 - lib/backburner/configuration.rb
@@ -131,19 +132,24 @@ files:
 - lib/backburner/version.rb
 - lib/backburner/worker.rb
 - lib/backburner/workers/simple.rb
+- lib/backburner/workers/threads_on_fork.rb
+- test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb
 - test/fixtures/hooked.rb
+- test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_jobs.rb
+- test/helpers/templogger.rb
 - test/helpers_test.rb
 - test/hooks_test.rb
 - test/job_test.rb
 - test/logger_test.rb
 - test/performable_test.rb
 - test/queue_test.rb
-- test/simple_worker_test.rb
 - test/test_helper.rb
 - test/worker_test.rb
+- test/workers/simple_worker_test.rb
+- test/workers/threads_on_fork_test.rb
 homepage: http://github.com/nesquena/backburner
 licenses: []
 post_install_message: 
@@ -169,17 +175,21 @@ signing_key:
 specification_version: 3
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra
 test_files:
+- test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb
 - test/fixtures/hooked.rb
+- test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_jobs.rb
+- test/helpers/templogger.rb
 - test/helpers_test.rb
 - test/hooks_test.rb
 - test/job_test.rb
 - test/logger_test.rb
 - test/performable_test.rb
 - test/queue_test.rb
-- test/simple_worker_test.rb
 - test/test_helper.rb
 - test/worker_test.rb
+- test/workers/simple_worker_test.rb
+- test/workers/threads_on_fork_test.rb
 has_rdoc: