beanstalk_farmer 0.2.2 → 0.3.0

data/Changelog.md

@@ -1,3 +1,22 @@
+## 0.3.0
+
+* Allow for job handlers to return instructions on how to complete the job at
+  the queue level. Please see README.md for details.
+
+## 0.2.3
+
+* Refactored many objects to not use Singleton pattern. This was introducing
+  global state. This is not a good thing.
+* Moved more behaviour into the Service layer.
+* Improvements to test environment.
+
+## 0.2.2
+
+* Test environment improvments for OS X.
+* Reserve timeouts handled correctly.
+* Better errors. This may or may not break your app. Please test carefully.
+* Support timeouts when reserving and working jobs in the Runner.
+
 ## 0.2.1
 
 * Fix malformed gemspec. Gem should actually work now.

data/README.md

@@ -12,6 +12,12 @@ la Rack (e.g. `Proc`s, `procs`s). The payload of the job will be passed into
 the arguments of this method. Bear in mind that the payload for a Beanstalk job
 is de-serialized JSON.
 
+By default all jobs will be deleted from the Beanstalk queue once completed.
+This can be configured and changed by returning an Array of arguments from the
+`#call` method. The first argument is expected to be the name of the method you
+wish to invoke on a job (see `Beanstalk::Job`). All remaining arguments will be
+passed to the named method.
+
 For usage examples, see the examples directory in the source code.
 
 ##  Development

data/lib/beanstalk_farmer/job.rb

@@ -5,7 +5,7 @@ module BeanstalkFarmer
   class Job
     include BeanstalkFarmer
 
-    attr_accessor :name, :args, :job
+    attr_accessor :name, :args, :job, :response
 
     # @param [Beanstalk::Job] job A Beanstalk job that has been reserved to be
     #   worked upon
@@ -15,22 +15,18 @@ module BeanstalkFarmer
       logger.info "JOB (#{name}) reserved"
     end
 
+    # @return [#call] a class used to process the reserved job
+    def handler
+      self.class.handler_pool[name]
+    end
+
     # Performs work for this job
     def work
-      # Stalker, the inspiration for this project, subtracted 1 from the job's
-      # TTR value. I'm not sure why at this point in time. Maybe to compensate
-      # for Job setup time.
-      Timeout.timeout(job.ttr) do
-        logger.info "JOB (#{name}) working"
-        handler = self.class.handler_pool[name]
-        handler.call(args)
-      end
+      do_some_work
     rescue Timeout::Error
-      logger.error "JOB (#{name}) out of time"
-      raise TimedOut, "#{name} could not finish in #{job.ttr} seconds"
+      handle_timeout!
     ensure
-      job.delete
-      logger.info "JOB (#{name}) done"
+      handle_job_response
     end
 
     # A pool of job handlers that can work on jobs in our queue
@@ -49,5 +45,30 @@ module BeanstalkFarmer
     def set_name_and_arguments
       self.name, self.args = MultiJson.decode(job.body)
     end
+
+    def do_some_work
+      Timeout.timeout(job.ttr) do
+        logger.info "JOB (#{name}) working"
+        self.response = handler.call(args)
+      end
+    end
+
+    def handle_timeout!
+      logger.error "JOB (#{name}) out of time"
+      raise TimedOut, "#{name} could not finish in #{job.ttr} seconds"
+    end
+
+    def handle_job_response
+      if response.is_a?(Array)
+        logger.info "JOB (#{name}) done. Response: #{response.inspect}"
+        job.public_send(*response)
+      else
+        logger.info "JOB (#{name}) done. Deleting by default."
+        job.delete
+      end
+    rescue NoMethodError => e
+      logger.info "JOB (#{name}) done. Assuming delete from response: #{response.inspect}"
+      job.delete
+    end
   end
 end

data/lib/beanstalk_farmer/runner.rb

@@ -1,10 +1,16 @@
-require 'singleton'
-
 module BeanstalkFarmer
-  # Maps tube names to job handlers, and manages the run loop
+  # Maps tube names to job handlers, and manages the run loop.
+  #
+  # @example
+  #
+  #   runner = BeanstalkFarmer::Runner.new
+  #
+  #   runner.register_handlers do
+  #     tube 'my_tube', lambda { |args| puts args {
+  #   end
+  #
+  #   runner.run!
   class Runner
-    include Singleton
-
     # @see Farmer::Job.handler_pool
     def handler_pool
       Job.handler_pool
@@ -12,7 +18,7 @@ module BeanstalkFarmer
 
     # @return [Farmer::BeanstalkService] a connection to the Beanstalk queue
     def service
-      @service ||= Service.instance
+      @service ||= Service.new
     end
 
     # Prepares tubes for watching
@@ -25,35 +31,36 @@ module BeanstalkFarmer
     # @param [Integer] timeout (nil) The optional timeout to use when
     #   reserving a job
     def reserve_and_work_job(timeout=nil)
-      job = service.reserve(timeout)
-      job.work
+      service.reserve_and_work_job(timeout)
     end
 
     # Runs a loop looking for jobs to reserve and run
     def work_jobs
-      loop { reserve_and_work_job }
+      service.run_loop { reserve_and_work_job }
     end
 
     # Closes the connection to the Beanstalk queue
-    def self.close_connection
-      instance.service.close
+    def close_connection
+      service.close
     end
 
     # @yield block A DSL to define jobs for your queue
     #
     # @example
-    #   Farmer::Runner.register_handlers do
+    #   runner = Farmer::Runner.new
+    #
+    #   runner.register_handlers do
     #     tube 'sms',   proc { |args| puts args }
     #     tube 'mine',  Proc.new { |args| puts args }
     #   end
-    def self.register_handlers(&block)
+    def register_handlers(&block)
       DSL.execute!(&block)
     end
 
     # Looks for jobs to reserve, and applies handlers to them
-    def self.run!
-      instance.prep_tubes
-      instance.work_jobs
+    def run!
+      prep_tubes
+      work_jobs
     end
   end
 end

data/lib/beanstalk_farmer/service.rb

@@ -1,19 +1,19 @@
 require 'beanstalk-client'
 require 'multi_json'
-require 'singleton'
 
 module BeanstalkFarmer
   ##
   # Provides an abstraction against our Beanstalk client to buffer us against
   # changes in their API.
   #
+  # Set the host and port through the configuration interface.
+  #
   # @todo Abstract all 3rd party errors
   # @todo Catch all 3rd party errors, and re-raise our own
   #
   # @private
   class Service
     include BeanstalkFarmer
-    include Singleton
 
     DEFAULT_DELAY    = 0
     DEFAULT_PRIORITY = 65536
@@ -26,9 +26,9 @@ module BeanstalkFarmer
     # @param [String] host (DEFAULT_HOST) the host name to of your Beanstalk queue
     #
     # @param [String] port (DEFAULT_PORT) the port that your Beanstalk queue is on
-    def initialize(host=Config.host, port=Config.port)
-      self.host = host
-      self.port = port
+    def initialize
+      self.host = Config.host
+      self.port = Config.port
     end
 
     # @return [String] a formatted URI for a Beanstalk queue
@@ -90,6 +90,17 @@ module BeanstalkFarmer
       raise TimedOut, e.message
     end
 
+    # @see BeanstalkFarmer::Service#reserve
+    def reserve_and_work_job(timeout=nil)
+      job = reserve(timeout)
+      job.work
+    end
+
+    # @yield A block to execute in the run loop
+    def run_loop(&block)
+      loop &block
+    end
+
     private
 
     def build_uri(host, port)

data/lib/beanstalk_farmer/version.rb

@@ -1,3 +1,3 @@
 module BeanstalkFarmer
-  VERSION = "0.2.2"
+  VERSION = '0.3.0'
 end

data/spec/beanstalk_farmer/job_spec.rb

@@ -1,17 +1,23 @@
 require 'spec_helper'
 
 describe BeanstalkFarmer::Job do
+  let(:name) { 'bacon' }
+
   let(:job) {
-    double('job', body: "[\"bacon\",{\"msg\":\"Hello\"}]", name: 'bacon', args: { 'msg' => 'Hello' }, ttr: 56, delete: true)
+    double('job', body: "[\"#{name}\",{\"msg\":\"Hello\"}]", name: name, args: { 'msg' => 'Hello' }, ttr: 56, delete: true)
   }
 
+  let(:delete_handler) { lambda { |args| [:delete] } }
+
+  let(:release_handler) { lambda { |args| [:release, 3, 240] } }
+
   describe '.new' do
     it 'sets the job' do
       described_class.new(job).job.should == job
     end
 
     it 'decodes the name of the job' do
-      described_class.new(job).name.should == job.name
+      described_class.new(job).name.should == name
     end
 
     it 'decodes the arguments for the job' do
@@ -31,27 +37,109 @@ describe BeanstalkFarmer::Job do
     end
   end
 
-  describe '#work' do
-    let(:handler) { proc { |args| args } }
+  describe '#handler' do
+    let(:processor) { double(:processor, call: true) }
 
     subject { described_class.new(job) }
 
-    before(:each) do
-      described_class.handler_pool[job.name] = handler
+    before do
+      described_class.handler_pool[name] = processor
     end
 
-    it 'performs some work for the handler' do
-      subject.work.should == job.args
+    after do
+      described_class.handler_pool(true)
     end
 
-    it 'times out if the job has run out of time' do
-      Timeout.should_receive(:timeout).with(job.ttr) { raise Timeout::Error }
-      expect { subject.work }.to raise_error(BeanstalkFarmer::TimedOut)
+    it 'is the handler class used to process the job' do
+      subject.handler.should == processor
     end
+  end
+
+  describe '#work' do
+    let(:connection) { double('Connection', job_stats: { ttr: 3 }) }
+
+    let(:name) { 'my_job' }
+
+    let(:body) { "[\"#{name}\",{\"msg\":\"Hello\"}]" }
+
+    after(:each) do
+      described_class.handler_pool(:reset)
+    end
+
+    context '(general behaviour)' do
+      let(:handler) { double('handler', call: true) }
+
+      let(:job) { double('job', body: body, ttr: 3, delete: true) }
+
+      subject { described_class.new(job) }
+
+      before do
+        described_class.handler_pool[name] = handler
+      end
+
+      it 'uses the handler to perform some work' do
+        handler.should_receive(:call)
+        subject.work
+      end
+
+      it 'times out if the handler cannot complete in time' do
+        handler.should_receive(:call).and_raise(Timeout::Error)
+
+        expect {
+          subject.work
+        }.to raise_error(BeanstalkFarmer::TimedOut)
+      end
+
+      it 'deletes the job by default' do
+        job.should_receive(:delete).once
+        subject.work
+      end
+    end
+
+    context '(response handling with no arguments)' do
+      let(:action) { :delete }
+
+      let(:handler) { lambda { |args| [action] } }
+
+      let(:job_id) { 3 }
+
+      let(:job) { Beanstalk::Job.new(connection, job_id, body) }
+
+      subject { described_class.new(job) }
+
+      before do
+        described_class.handler_pool[name] = handler
+      end
+
+      it 'calls the specified action' do
+        connection.should_receive(:delete).with(job_id)
+        subject.work
+      end
+    end
+
+    context 'response handling with arguments' do
+      let(:action) { :release }
+
+      let(:new_priority) { 3 }
+
+      let(:new_delay) { 240 }
+
+      let(:handler) { lambda { |args| [action, new_priority, new_delay] } }
+
+      let(:job_id) { 1 }
+
+      let(:job) { Beanstalk::Job.new(connection, job_id, body) }
+
+      subject { described_class.new(job) }
+
+      before do
+        described_class.handler_pool[name] = handler
+      end
 
-    it 'deletes the job' do
-      subject.job.should_receive(:delete)
-      subject.work
+      it 'reserves the job with appropriate arguments' do
+        connection.should_receive(action).with(job_id, new_priority, new_delay)
+        subject.work
+      end
     end
   end
 end

data/spec/beanstalk_farmer/runner_spec.rb

@@ -1,11 +1,7 @@
 require 'spec_helper'
 
 describe BeanstalkFarmer::Runner do
-  subject { described_class.instance }
-
-  it 'is a singleton' do
-    described_class.included_modules.should include(Singleton)
-  end
+  subject { described_class.new }
 
   its(:service) { should be_a_kind_of(BeanstalkFarmer::Service) }
 
@@ -24,24 +20,15 @@ describe BeanstalkFarmer::Runner do
   end
 
   describe '#reserve_and_work_job' do
-    let(:job) { double('Farmer::Job', work: true) }
-
-    before(:each) do
-      subject.stub_chain(:service, :reserve) { job }
-    end
-
-    it 'reserves a job' do
-      subject.service.should_receive(:reserve) { job }
-      subject.reserve_and_work_job
-    end
+    let(:timeout) { 45 }
 
-    it 'works the job' do
-      job.should_receive(:work)
-      subject.reserve_and_work_job
+    it 'tells the service to reserve and work a job' do
+      subject.service.should_receive(:reserve_and_work_job).with(timeout)
+      subject.reserve_and_work_job(timeout)
     end
   end
 
-  describe '.run!' do
+  describe '#run!' do
     before(:each) do
       subject.stub(:prep_tubes)
       subject.stub(:work_jobs)
@@ -49,19 +36,19 @@ describe BeanstalkFarmer::Runner do
 
     it 'prepares tubes for watching' do
       subject.should_receive(:prep_tubes)
-      described_class.run!
+      subject.run!
     end
 
     it 'works jobs' do
       subject.should_receive(:work_jobs)
-      described_class.run!
+      subject.run!
     end
   end
 
-  describe '.close_connection' do
+  describe '#close_connection' do
     it 'closes the connection to the service' do
       subject.service.should_receive(:close)
-      described_class.close_connection
+      subject.close_connection
     end
   end
 end

data/spec/beanstalk_farmer/service_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe BeanstalkFarmer::Service, :beanstalk_required do
-  subject { described_class.instance }
+  subject { described_class.new }
 
   its(:host) { should == BeanstalkFarmer::Config.host }
 
@@ -14,14 +14,25 @@ describe BeanstalkFarmer::Service, :beanstalk_required do
   end
 
   describe '#connection' do
-    after(:each) do
-      subject.close
+    context 'when successful' do
+      after(:each) do
+        subject.close
+      end
+
+      it 'establishes a connection to Beanstalk' do
+        expect {
+          subject.connection.use 'fake.tube'
+        }.to_not raise_error(BeanstalkFarmer::NotConnectedError)
+      end
     end
 
-    it 'establishes a connection to Beanstalk' do
-      expect {
-        subject.connection.use 'fake.tube'
-      }.to_not raise_error(BeanstalkFarmer::NotConnectedError)
+    context 'when failing' do
+      it 'raises an exception if a connection cannot be established' do
+        Beanstalk::Pool.stub(:new).and_raise(Beanstalk::NotConnected)
+        expect {
+          subject.connection
+        }.to raise_error(BeanstalkFarmer::NotConnectedError)
+      end
     end
   end
 
@@ -89,4 +100,27 @@ describe BeanstalkFarmer::Service, :beanstalk_required do
       }.to raise_error(BeanstalkFarmer::TimedOut)
     end
   end
+
+  describe '#reserve_and_work_job' do
+    let(:job) { double(work: true) }
+
+    before(:each) do
+      subject.stub(:reserve) { job }
+    end
+
+    it 'reserves a job' do
+      subject.should_receive(:reserve) { job }
+      subject.reserve_and_work_job
+    end
+
+    it 'passes an optional timeout to the reserved job' do
+      subject.should_receive(:reserve).with(30) { job }
+      subject.reserve_and_work_job(30)
+    end
+
+    it 'works the reserved job' do
+      job.should_receive(:work)
+      subject.reserve_and_work_job
+    end
+  end
 end

data/spec/integration/special_worker_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+class ReleaseWorker
+  LIMIT = 2
+
+  @@count = 0
+
+  def self.call(args)
+    if @@count < LIMIT
+      $stderr << "Count: #{@@count}"
+      @@count += 1
+      [:release]
+    else
+      $stderr << args
+    end
+  end
+end unless defined?(ReleaseWorker)
+
+describe 'Workers do work', :beanstalk_required do
+  let!(:service) { BeanstalkFarmer::Service.new }
+
+  let!(:runner) { BeanstalkFarmer::Runner.new }
+
+  let(:final_message) { 'Hello. Finally done.' }
+
+  before(:each) do
+    @old_stderr = $stderr
+    @capture    = StringIO.new
+    $stderr     = @capture
+
+    runner.register_handlers do
+      tube 'echo.release', ReleaseWorker
+    end
+
+    runner.prep_tubes
+    service.enqueue 'echo.release', final_message
+  end
+
+  after(:each) do
+    $stderr  = @old_stderr
+    @capture = nil
+    service.close
+  end
+
+  it 'does work with special post-job effects' do
+    ReleaseWorker::LIMIT.times do |n|
+      runner.reserve_and_work_job
+
+      if n < ReleaseWorker::LIMIT
+        @capture.string.should == "Count: #{n}"
+        @capture.rewind
+      else
+        @capture.string.should == final_message
+      end
+    end
+  end
+end

data/spec/integration/working_spec.rb

@@ -7,59 +7,59 @@ class ClassWorker
 end unless defined?(ClassWorker)
 
 describe 'Workers do work', :beanstalk_required do
+  let!(:service) { BeanstalkFarmer::Service.new }
+
+  let!(:runner) { BeanstalkFarmer::Runner.new }
+
   before(:each) do
     @old_stderr = $stderr
     @capture    = StringIO.new
     $stderr     = @capture
 
-    BeanstalkFarmer::Runner.register_handlers do
+    runner.register_handlers do
       tube 'echo.small_proc', proc { |args| $stderr << args }
       tube 'echo.big_proc',   Proc.new { |args| $stderr << args }
       tube 'echo.class',      ClassWorker
       tube 'echo.block',      do |args| $stderr << args; end
     end
 
-    BeanstalkFarmer::Runner.instance.prep_tubes
+    runner.prep_tubes
   end
 
   after(:each) do
     $stderr  = @old_stderr
     @capture = nil
-    BeanstalkFarmer::Service.instance.close
+    service.close
   end
 
   it 'performs enqueued work with small procs' do
     message = 'Hello, proc'
-
-    BeanstalkFarmer::Service.instance.enqueue 'echo.small_proc', message
-    BeanstalkFarmer::Runner.instance.reserve_and_work_job
+    service.enqueue 'echo.small_proc', message
+    runner.reserve_and_work_job
 
     @capture.string.should == message
   end
 
   it 'performs enqueued work with blocks' do
     message = 'Hello, block'
-
-    BeanstalkFarmer::Service.instance.enqueue 'echo.block', message
-    BeanstalkFarmer::Runner.instance.reserve_and_work_job
+    service.enqueue 'echo.block', message
+    runner.reserve_and_work_job
 
     @capture.string.should == message
   end
 
   it 'performs enqueued work with big procs' do
     message = 'Hello, big Proc'
-
-    BeanstalkFarmer::Service.instance.enqueue 'echo.big_proc', message
-    BeanstalkFarmer::Runner.instance.reserve_and_work_job
+    service.enqueue 'echo.big_proc', message
+    runner.reserve_and_work_job
 
     @capture.string.should == message
   end
 
   it 'performs enqueued work with class' do
     message = 'Hello, Class'
-
-    BeanstalkFarmer::Service.instance.enqueue 'echo.class', message
-    BeanstalkFarmer::Runner.instance.reserve_and_work_job
+    service.enqueue 'echo.class', message
+    runner.reserve_and_work_job
 
     @capture.string.should == message
   end

data/spec/spec_helper.rb

@@ -9,6 +9,7 @@ SimpleCov.start do
 end
 
 require 'beanstalk_farmer'
+require 'beanstalk-client'
 
 Dir['./spec/support/**/*.rb'].each { |support_file| require support_file }
 

metadata

@@ -2,7 +2,7 @@
 name: beanstalk_farmer
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors: 
 - James Herdamn
@@ -10,10 +10,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-13 00:00:00 Z
+date: 2011-07-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: beanstalk-client
+  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -21,10 +22,10 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.1.0
   type: :runtime
-  prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: multi_json
+  prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -32,10 +33,10 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.0.2
   type: :runtime
-  prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rspec
+  prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -43,10 +44,10 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "2.6"
   type: :development
-  prerelease: false
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: rake
+  prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -54,10 +55,10 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.9.0
   type: :development
-  prerelease: false
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: simplecov
+  prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -65,7 +66,6 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.4.2
   type: :development
-  prerelease: false
   version_requirements: *id005
 description: Farmer is a nice little kit to manage a Beanstalk job queue
 email: 
@@ -94,6 +94,7 @@ files:
 - spec/beanstalk_farmer/runner_spec.rb
 - spec/beanstalk_farmer/service_spec.rb
 - spec/beastalk_farmer_spec.rb
+- spec/integration/special_worker_spec.rb
 - spec/integration/working_spec.rb
 - spec/spec_helper.rb
 - spec/support/have_tube_named.rb
@@ -110,18 +111,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -1883747192443942397
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -1883747192443942397
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
@@ -137,6 +132,7 @@ test_files:
 - spec/beanstalk_farmer/runner_spec.rb
 - spec/beanstalk_farmer/service_spec.rb
 - spec/beastalk_farmer_spec.rb
+- spec/integration/special_worker_spec.rb
 - spec/integration/working_spec.rb
 - spec/spec_helper.rb
 - spec/support/have_tube_named.rb