backburner 0.3.0 → 0.3.1

data/CHANGELOG.md

@@ -1,6 +1,10 @@
 # CHANGELOG
 
-## Version 0.3.1 (Unreleased)
+# Version 0.3.2 (Unreleased)
+
+## Version 0.3.1 (Dec 28 2012)
+
+ * Adds basic forking processing strategy and rake tasks (Thanks @danielfarrell)
 
 ## Version 0.3.0 (Nov 14 2012)
 

data/README.md

@@ -254,21 +254,22 @@ This is why our examples use object IDs instead of passing around objects.
 
 ### Processing Strategies
 
-In Backburner, there are actually multiple different strategies for processing jobs
-which are reflected by multiple workers.
+In Backburner, there are several different strategies for processing jobs
+which are reflected by multiple worker subclasses.
 Custom workers can be [defined fairly easily](https://github.com/nesquena/backburner/wiki/Defining-Workers).
 By default, Backburner comes with the following workers built-in:
 
 | Worker | Description                                                                 |
 | ------- | -------------------------------                                            |
 | `Backburner::Workers::Simple` | Single threaded, no forking worker. Simplest option. |
+| `Backburner::Workers::Forking` | Basic forking worker that manages crashes and memory bloat. |
 | `Backburner::Workers::ThreadsOnFork` | Forking worker that utilizes threads for concurrent processing. |
 
 You can select the default worker for processing with:
 
 ```ruby
 Backburner.configure do |config|
-  config.default_worker = Backburner::Workers::Simple
+  config.default_worker = Backburner::Workers::Forking
 end
 ```
 
@@ -400,6 +401,7 @@ jobs processed by your beanstalk workers. An excellent addition to your Backburn
  * [Tim Lee](https://github.com/timothy1ee), [Josh Hull](https://github.com/joshbuddy), [Nico Taing](https://github.com/Nico-Taing) - Helping me work through the idea
  * [Miso](http://gomiso.com) - Open-source friendly place to work
  * [Renan T. Fernandes](https://github.com/ShadowBelmolve) - Added threads_on_fork worker
+ * [Daniel Farrell](https://github.com/danielfarrell) - Added forking worker
 
 ## Contributing
 

data/examples/stress.rb

@@ -26,5 +26,6 @@ end
   Backburner.enqueue TestJob, i
 end
 
-# Work tasks using threaded worker
-Backburner.work("test-job", :worker => Backburner::Workers::ThreadsOnFork)
+# Work tasks using threads_on_fork worker
+# twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
+Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)

data/lib/backburner.rb

@@ -11,6 +11,7 @@ require 'backburner/hooks'
 require 'backburner/performable'
 require 'backburner/worker'
 require 'backburner/workers/simple'
+require 'backburner/workers/forking'
 require 'backburner/workers/threads_on_fork'
 require 'backburner/queue'
 

data/lib/backburner/tasks.rb

@@ -18,6 +18,15 @@ namespace :backburner do
     end
   end # simple
 
+  namespace :forking do
+    # QUEUE=foo,bar,baz rake backburner:forking:work
+    desc "Starts backburner worker using fork processing"
+    task :work => :environment do
+      queues = (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
+      Backburner.work queues, :worker => Backburner::Workers::Forking
+    end
+  end # forking
+
   namespace :threads_on_fork do
     # QUEUE=twitter:10:5000:5,parse_page,send_mail,verify_bithday THREADS=2 GARBAGE=1000 rake backburner:threads_on_fork:work
     # twitter tube will have 10 threads, garbage after 5k executions and retry 5 times.

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/backburner/workers/forking.rb

@@ -0,0 +1,48 @@
+module Backburner
+  module Workers
+    class Forking < Worker
+      # Used to prepare job queues before processing jobs.
+      # Setup beanstalk tube_names and watch all specified tubes for jobs.
+      #
+      # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
+      # @example
+      #   @worker.prepare
+      #
+      def prepare
+        self.tube_names.map! { |name| expand_tube_name(name)  }
+        log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+        self.connection.tubes.watch!(*self.tube_names)
+      end
+
+      # Starts processing new jobs indefinitely.
+      # Primary way to consume and process jobs in specified tubes.
+      #
+      # @example
+      #   @worker.start
+      #
+      def start
+        prepare
+        loop { fork_one_job }
+      end
+
+      # Need to re-establish the connection to the server(s) after forking
+      # Waits for a job, works the job, and exits
+      def fork_one_job
+        pid = Process.fork do
+          @connection = Connection.new(Backburner.configuration.beanstalk_url)
+          work_one_job
+          coolest_exit
+        end
+        Process.wait(pid)
+      end
+
+      # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
+      # parent process
+      # We will use exitcode 99 that means the fork reached the garbage number
+      def coolest_exit
+        Kernel.exit! 99
+      end
+
+    end
+  end
+end

data/test/fixtures/test_forking_jobs.rb

@@ -0,0 +1,56 @@
+class ResponseForkingJob
+  include Backburner::Queue
+  queue_priority 1000
+  def self.perform(data)
+    $worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
+    $worker_success = data['worker_success'] if data['worker_success']
+    $worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
+    $worker_raise = data['worker_raise'] if data['worker_raise']
+  end
+end
+
+class TestJobForking
+  include Backburner::Queue
+  queue_priority 1000
+  def self.perform(x, y)
+    Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
+        :worker_test_count_set => x + y
+    }], :queue => 'response'
+  end
+end
+
+class TestFailJobForking
+  include Backburner::Queue
+  def self.perform(x, y)
+    Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
+       :worker_raise => true
+    }], :queue => 'response'
+  end
+end
+
+class TestRetryJobForking
+  include Backburner::Queue
+  def self.perform(x, y)
+    if $worker_test_count <= 2
+      Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
+          :worker_test_count => 1
+      }], :queue => 'response'
+
+      raise RuntimeError
+    else # succeeds
+      Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
+          :worker_test_count => 1,
+          :worker_success => true
+      }], :queue => 'response'
+    end
+  end
+end
+
+class TestAsyncJobForking
+  include Backburner::Performable
+  def self.foo(x, y)
+    Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
+        :worker_test_count_set => x * y
+    }], :queue => 'response'
+  end
+end

data/test/workers/forking_worker_test.rb

@@ -0,0 +1,179 @@
+require File.expand_path('../../test_helper', __FILE__)
+require File.expand_path('../../fixtures/test_forking_jobs', __FILE__)
+
+describe "Backburner::Workers::Forking module" do
+
+  before do
+    Backburner.default_queues.clear
+    @worker_class = Backburner::Workers::Forking
+  end
+
+  describe "for prepare method" do
+    it "should watch specified tubes" do
+      worker = @worker_class.new(["foo", "bar"])
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.foo", "demo.test.bar"], @worker_class.connection.tubes.watched.map(&:name)
+      assert_match /demo\.test\.foo/, out
+    end # multiple
+
+    it "should watch single tube" do
+      worker = @worker_class.new("foo")
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo"], worker.tube_names
+      assert_same_elements ["demo.test.foo"], @worker_class.connection.tubes.watched.map(&:name)
+      assert_match /demo\.test\.foo/, out
+    end # single
+
+    it "should respect default_queues settings" do
+      Backburner.default_queues.concat(["foo", "bar"])
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.foo", "demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.foo", "demo.test.bar"], @worker_class.connection.tubes.watched.map(&:name)
+      assert_match /demo\.test\.foo/, out
+    end
+
+    it "should assign based on all tubes" do
+      @worker_class.any_instance.expects(:all_existing_queues).once.returns("bar")
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_equal ["demo.test.bar"], worker.tube_names
+      assert_same_elements ["demo.test.bar"], @worker_class.connection.tubes.watched.map(&:name)
+      assert_match /demo\.test\.bar/, out
+    end # all assign
+
+    it "should properly retrieve all tubes" do
+      worker = @worker_class.new
+      out = capture_stdout { worker.prepare }
+      assert_contains worker.tube_names, "demo.test.test-job"
+      assert_contains @worker_class.connection.tubes.watched.map(&:name), "demo.test.test-job"
+      assert_match /demo\.test\.test-job/, out
+    end # all read
+  end # prepare
+
+  describe "for fork_one_job method" do
+
+    it "should fork, reconnect, work job, and exit" do
+      clear_jobs!("bar.foo")
+      @worker_class.enqueue TestJobForking, [1, 2], :queue => "bar.foo"
+
+      fake_pid = 45
+      Process.expects(:fork).returns(fake_pid) do |&block|
+        Connection.expects(:new).with(Backburner.configuration.beanstalk_url)
+        @worker_class.any_instance.expects(:work_one_job)
+        @worker_class.any_instance.expects(:coolest_exit)
+        block.call
+      end
+      Process.expects(:wait).with(fake_pid)
+
+      silenced(2) do
+        worker = @worker_class.new('bar.foo')
+        worker.prepare
+        worker.fork_one_job
+      end
+    end
+
+  end # fork_one_job
+
+  describe "practical tests" do
+
+    before do
+      @templogger = Templogger.new('/tmp')
+      Backburner.configure { |config| config.logger = @templogger.logger }
+      $worker_test_count = 0
+      $worker_success = false
+      $worker_raise   = false
+      clear_jobs!('response')
+      clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
+      silenced do
+        @response_worker = @worker_class.new('response')
+      end
+    end
+
+    after do
+      @templogger.close
+      clear_jobs!('response')
+      clear_jobs!('bar.foo.1', 'bar.foo.2', 'bar.foo.3', 'bar.foo.4', 'bar.foo.5')
+    end
+
+
+    it "should work an enqueued job" do
+      @worker = @worker_class.new('bar.foo.1')
+      @worker_class.enqueue TestJobForking, [1, 2], :queue => "bar.foo.1"
+      @worker.prepare
+      silenced(2) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Completed TestJobFork/m)
+        @response_worker.prepare
+        @response_worker.work_one_job
+      end
+      assert_equal 3, $worker_test_count
+    end # enqueue
+
+    it "should work for an async job" do
+      @worker = @worker_class.new('bar.foo.2')
+      TestAsyncJobForking.async(:queue => 'bar.foo.2').foo(3, 5)
+      @worker.prepare
+      silenced(4) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Completed TestAsyncJobFork/m)
+        @response_worker.prepare
+        @response_worker.work_one_job
+      end
+      assert_equal 15, $worker_test_count
+    end # async
+
+    it "should fail quietly if there's an argument error" do
+      Backburner.configure { |config| config.max_job_retries = 0 }
+      @worker = @worker_class.new('bar.foo.3')
+      @worker_class.enqueue TestJobForking, ["bam", "foo", "bar"], :queue => "bar.foo.3"
+      @worker.prepare
+      silenced(5) do
+        @worker.fork_one_job
+        @templogger.wait_for_match(/Finished TestJobFork.*attempt 1 of 1/m)
+      end
+      assert_match(/Exception ArgumentError/, @templogger.body)
+      assert_equal 0, $worker_test_count
+    end # fail, argument
+
+    it "should support retrying jobs and burying" do
+      Backburner.configure { |config| config.max_job_retries = 1; config.retry_delay = 0 }
+      @worker = @worker_class.new('bar.foo.4')
+      @worker_class.enqueue TestRetryJobForking, ["bam", "foo"], :queue => 'bar.foo.4'
+      @worker.prepare
+      silenced(4) do
+        2.times do
+          $worker_test_count += 1
+          @worker.fork_one_job
+        end
+        @templogger.wait_for_match(/Finished TestRetryJobFork.*attempt 2 of 2/m)
+        @response_worker.prepare
+        2.times { @response_worker.work_one_job }
+      end
+      assert_equal 4, $worker_test_count
+      assert_equal false, $worker_success
+    end # retry, bury
+
+    it "should support retrying jobs and succeeds" do
+      Backburner.configure { |config| config.max_job_retries = 2; config.retry_delay = 0 }
+      @worker = @worker_class.new('bar.foo.5')
+      @worker_class.enqueue TestRetryJobForking, ["bam", "foo"], :queue => 'bar.foo.5'
+      @worker.prepare
+      silenced(4) do
+        3.times do
+          $worker_test_count += 1
+          @worker.fork_one_job
+        end
+        @templogger.wait_for_match(/Completed TestRetryJobFork/m)
+        @response_worker.prepare
+        3.times { @response_worker.work_one_job }
+      end
+      assert_equal 6, $worker_test_count
+      assert_equal true, $worker_success
+    end # retrying, succeeds
+
+  end # practical tests
+
+
+end

data/test/workers/{threads_on_fork_test.rb → threads_on_fork_worker_test.rb}

@@ -50,6 +50,10 @@ describe "Backburner::Workers::ThreadsOnFork module" do
   end
 
   describe "for prepare method" do
+    before do
+      Backburner.configure { |config| config.logger = false }
+    end
+
     it "should watch specified tubes" do
       worker = @worker_class.new(["foo", "bar"])
       out = capture_stdout { worker.prepare }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-15 00:00:00.000000000 Z
+date: 2012-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -131,6 +131,7 @@ files:
 - lib/backburner/tasks.rb
 - lib/backburner/version.rb
 - lib/backburner/worker.rb
+- lib/backburner/workers/forking.rb
 - lib/backburner/workers/simple.rb
 - lib/backburner/workers/threads_on_fork.rb
 - test/async_proxy_test.rb
@@ -138,6 +139,7 @@ files:
 - test/connection_test.rb
 - test/fixtures/hooked.rb
 - test/fixtures/test_fork_jobs.rb
+- test/fixtures/test_forking_jobs.rb
 - test/fixtures/test_jobs.rb
 - test/helpers/templogger.rb
 - test/helpers_test.rb
@@ -148,8 +150,9 @@ files:
 - test/queue_test.rb
 - test/test_helper.rb
 - test/worker_test.rb
+- test/workers/forking_worker_test.rb
 - test/workers/simple_worker_test.rb
-- test/workers/threads_on_fork_test.rb
+- test/workers/threads_on_fork_worker_test.rb
 homepage: http://github.com/nesquena/backburner
 licenses: []
 post_install_message: 
@@ -180,6 +183,7 @@ test_files:
 - test/connection_test.rb
 - test/fixtures/hooked.rb
 - test/fixtures/test_fork_jobs.rb
+- test/fixtures/test_forking_jobs.rb
 - test/fixtures/test_jobs.rb
 - test/helpers/templogger.rb
 - test/helpers_test.rb
@@ -190,6 +194,6 @@ test_files:
 - test/queue_test.rb
 - test/test_helper.rb
 - test/worker_test.rb
+- test/workers/forking_worker_test.rb
 - test/workers/simple_worker_test.rb
-- test/workers/threads_on_fork_test.rb
-has_rdoc: 
+- test/workers/threads_on_fork_worker_test.rb