juggler 0.0.2 → 0.1.0

data/.gitignore

@@ -1,3 +1,3 @@
 .DS_Store
 pkg/*
-juggler.gemspec
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in foo.gemspec
+gemspec

data/README.md

@@ -2,17 +2,17 @@ Add jobs for asynchronous processing
 
     Juggler.throw(:method, params)
 
-Add handlers, with optional concurrency, inside and EM loop
+Add handlers, with optional concurrency, inside an EM loop
 
     EM.run {
-      Juggler.juggle(:method, 10) do |params|
-        # This code must return an eventmachine deferrable object
+      Juggler.juggle(:method, 10) do |deferrable, params|
+        # Succeed the deferrable when the job is done
       end
     }
 
 For example
 
-    Juggler.juggle(:download, 10) do |params|
+    Juggler.juggle(:download, 10) do |df, params|
       http = EM::Protocols::HttpClient.request({
         :host => params[:host], 
         :port => 80, 
@@ -20,6 +20,39 @@ For example
       })
       http.callback do |response|
         puts "Got response status #{response[:status]} for #{a}"
+        df.success
       end
-      http
     end
+
+The job is considered to have failed in the following cases:
+
+* If the block raises an error
+* If the block fails the passed deferrable
+* If the job timeout is exceeded. In this case the passed deferrable will be failed by juggler. If you need to clean up any state in this case (for example you might want to cancel a HTTP request) then you should bind to `df.errback`.
+
+## Customising behaviour
+
+### Customising the back-off for failed jobs
+
+By default, juggler will backoff jobs which failed exponentially using an exponent of 1.3, up to a maximum delay of 1 day, at which point the job will be buried. It's possible to customise this behaviour:
+
+    Juggler.backoff_function = lambda { |job_runner, job_stats|
+      # job_stats is a hash with string keys, as returned by beanstalkd's
+      # stats-job command. Particularly useful stats in this context are:
+      #
+      # job_stats["age"] - the time since the put command that created this job
+      # job_stats["delay"] - the previous amount of time delayed
+
+      new_delay = ([1, job_stats["delay"] * 2].max).ceil
+      if job_stats["age"] > 300
+        job_runner.delete
+        # Or you could bury the job
+        # job_runner.bury
+      else
+        job_runner.release(new_delay)
+      end
+    }
+
+## Important points to note
+
+* If your deferrable code raises errors, this will not be handled by juggler.

data/Rakefile

@@ -1,17 +1,10 @@
-require 'rubygems'
-require 'rake'
+require "bundler/gem_tasks"
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "juggler"
-    gem.summary = %Q{Juggling background jobs with EventMachine and Beanstalkd}
-    gem.description = %Q{Juggling background jobs with EventMachine and Beanstalkd}
-    gem.email = "me@mloughran.com"
-    gem.homepage = "http://github.com/mloughran/juggler"
-    gem.authors = ["Martyn Loughran"]
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |s|
+  s.pattern = 'spec/**/*.rb'
 end
+
+task :default => :spec
+task :test => :spec

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.1.0

data/examples/app.rb

@@ -5,3 +5,8 @@ get '/slow' do
   sleep 3
   return 'Finally done'
 end
+
+get '/fast' do
+  sleep 1
+  return 'Fast done'
+end

data/examples/test_consumer.rb

@@ -1,14 +1,13 @@
 require 'rubygems'
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'juggler'
-require 'eventmachine'
 
-Juggler.logger = Logger.new(STDOUT)
+Juggler.logger.level = Logger::DEBUG
 
 EM.run {
   Juggler.juggle(:http, 3) do |path|
     http = EM::Protocols::HttpClient.request({
-      :host => "localhost",
+      :host => "0.0.0.0",
       :port => 4567,
       :request => path
     })
@@ -22,7 +21,7 @@ EM.run {
   Juggler.juggle(:timer, 5) do |params|
     defer = EM::DefaultDeferrable.new
 
-    EM::Timer.new(1) do
+    EM::Timer.new(10) do
       defer.set_deferred_status :succeeded, nil
       # defer.set_deferred_status :failed, nil
     end

data/examples/test_producer.rb

@@ -2,19 +2,25 @@ require 'rubygems'
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'juggler'
 
-Juggler.logger = Logger.new(STDOUT)
-
 # Throw some jobs
 
-class Foo; end
+# EM.run {
+#   # 10.times do |i|
+#   #   path = ['/fast', '/slow'][i % 2]
+#   #   Juggler.throw(:http, path, :ttr => 40)
+#   # end
+#   1.times { Juggler.throw(:timer, {:foo => 'bar'}, :ttr => 5) }
+# }
 
-2.times do |i|
-  path = ['/fast', '/slow'][i % 2]
-  Juggler.throw(:http, path, {
-    :ttr => 1
-  })
+Thread.new do
+  EM.run
 end
-10.times { Juggler.throw(:timer, {:foo => 'bar'}) }
 
-# Test that unmarshaling errors are handled
-Juggler.throw(:http, Foo.new)
+loop do
+  puts "Choose your ttr!"
+  time = gets.to_i
+  puts "Creating job with ttr #{time}"
+  EM.next_tick {
+    Juggler.throw(:timer, {:foo => 'bar'}, :ttr => time)
+  }
+end

data/juggler.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name        = "juggler"
+  s.version     = "0.1.0"
+  s.authors     = ["Martyn Loughran"]
+  s.email       = ["me@mloughran.com"]
+  s.homepage    = "https://github.com/mloughran/juggler"
+  s.summary     = %q{Juggling background jobs with EventMachine and Beanstalkd}
+  s.description = %q{Juggling background jobs with EventMachine and Beanstalkd}
+
+  s.add_runtime_dependency "em-jack", "~> 0.1.0"
+
+  s.add_development_dependency "em-spec"
+  s.add_development_dependency "rake"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/juggler.rb

@@ -1,17 +1,31 @@
-require 'beanstalk-client'
-
-autoload :Logger, 'logger'
+require 'em-jack'
+require 'eventmachine'
+require 'uri'
 
 class Juggler
   class << self
-    attr_writer :hosts, :logger
+    attr_writer :logger
+    attr_writer :shutdown_grace_timeout
+    attr_accessor :exception_handler
+    attr_accessor :backoff_function
+
+    def server=(uri)
+      @server = URI.parse(uri)
+    end
 
-    def hosts
-      @hosts ||= ['localhost:11300']
+    def server
+      @server ||= URI.parse("beanstalk://localhost:11300")
+    end
+
+    # By default after receiving QUIT juggler will wait up to 2s for running
+    # jobs to complete before killing them
+    def shutdown_grace_timeout
+      @shutdown_grace_timeout || 2
     end
 
     def logger
       @logger ||= begin
+        require 'logger'
         logger = Logger.new(STDOUT)
         logger.level = Logger::WARN
         logger.debug("Created logger")
@@ -22,15 +36,14 @@ class Juggler
     def throw(method, params, options = {})
       # TODO: Do some checking on the method
       connection.use(method.to_s)
-
-      priority = options[:priority] || 50
-      delay = 0
-      # Add 2s because we want to handle the timeout before beanstalk does
-      ttr = (options[:ttr] || 60) + 2
-
-      connection.put(Marshal.dump(params), priority, delay, ttr)
+      connection.put(Marshal.dump(params), options)
     end
 
+    # Strategy block: should return a deferrable object (so that juggler can 
+    # apply callbacks and errbacks). You should note that this deferrable may 
+    # be failed by juggler if the job timeout is exceeded, and therefore you 
+    # are responsible for cleaning up your state (for example cancelling any 
+    # timers which you have created)
     def juggle(method, concurrency = 1, &strategy)
       Runner.new(method, concurrency, strategy).run
     end
@@ -38,9 +51,30 @@ class Juggler
     private
 
     def connection
-      @connection ||= Beanstalk::Pool.new(hosts)
+      @connection ||= EMJack::Connection.new({
+        :host => server.host,
+        :port => server.port
+      })
     end
   end
 end
 
+# Default exception handler
+Juggler.exception_handler = Proc.new do |e|
+  Juggler.logger.error "Error running job: #{e.message} (#{e.class})"
+  Juggler.logger.debug e.backtrace.join("\n")
+end
+
+# Default backoff function
+Juggler.backoff_function = Proc.new do |job_runner, job_stats|
+  # 2, 3, 4, 6, 8, 11, 15, 20, ..., 72465
+  delay = ([1, job_stats["delay"] * 1.3].max).ceil
+  if delay > 60 * 60 * 24
+    job_runner.bury
+  else
+    job_runner.release(delay)
+  end
+end
+
 Juggler.autoload 'Runner', 'juggler/runner'
+Juggler.autoload 'JobRunner', 'juggler/job_runner'

data/lib/juggler/job_runner.rb

@@ -0,0 +1,144 @@
+require 'juggler/state_machine'
+
+class Juggler
+  class JobRunner
+    include StateMachine
+    
+    state :new
+    state :running, :pre => :fetch_stats, :enter => :run_strategy
+    state :succeeded, :enter => :delete
+    state :timed_out, :enter => [:timeout_strategy, :backoff]
+    state :failed, :enter => :delete
+    state :retried, :enter => :backoff
+    state :done
+    
+    attr_reader :job
+    
+    def initialize(job, params, strategy)
+      @job = job
+      @params = params
+      @strategy = strategy
+      Juggler.logger.debug {
+        "#{to_s}: New job with body: #{params.inspect}"
+      }
+      @_state = :new
+    end
+    
+    def run
+      change_state(:running)
+    end
+    
+    def check_for_timeout
+      if state == :running
+        if (time_left = @end_time - Time.now) < 1
+          Juggler.logger.info("#{to_s}: Timed out (#{time_left}s left)")
+          change_state(:timed_out)
+        end
+      end
+    end
+    
+    def to_s
+      "Job #{@job.jobid}"
+    end
+    
+    def release(delay = 0)
+      Juggler.logger.debug { "#{to_s}: releasing" }
+      release_def = job.release(:delay => delay)
+      release_def.callback {
+        Juggler.logger.info { "#{to_s}: released for retry in #{delay}s" }
+        change_state(:done)
+      }
+      release_def.errback {
+        Juggler.logger.error { "#{to_s}: release failed (could not release)" }
+        change_state(:done)
+      }
+    end
+
+    def bury
+      Juggler.logger.warn { "#{to_s}: burying" }
+      release_def = job.bury(100000) # Set priority till em-jack fixed
+      release_def.callback {
+        change_state(:done)
+      }
+      release_def.errback {
+        change_state(:done)
+      }
+    end
+
+    def delete
+      dd = job.delete
+      dd.callback do
+        Juggler.logger.debug "#{to_s}: deleted"
+        change_state(:done)
+      end
+      dd.errback do
+        Juggler.logger.debug "#{to_s}: delete operation failed"
+        change_state(:done)
+      end
+    end
+
+    private
+    
+    # Retrives job stats from beanstalkd
+    def fetch_stats
+      dd = EM::DefaultDeferrable.new
+
+      Juggler.logger.debug { "#{to_s}: Fetching stats" }
+
+      stats_def = job.stats
+      stats_def.callback do |stats|
+        @stats = stats
+        @end_time = Time.now + stats["time-left"]
+        Juggler.logger.debug { "#{to_s} stats: #{stats.inspect}"}
+        dd.succeed
+      end
+      stats_def.errback {
+        Juggler.logger.error { "#{to_s}: Fetching stats failed" }
+        dd.fail
+      }
+
+      dd
+    end
+
+    # Wraps running the actual job.
+    # Returns a deferrable that fails if there is an exception calling the 
+    # strategy or if the strategy triggers errback
+    def run_strategy
+      begin
+        sd = EM::DefaultDeferrable.new
+        @strategy.call(sd, @params)
+        sd.callback {
+          change_state(:succeeded)
+        }
+        sd.errback { |e|
+          # timed_out error is already handled
+          next if e == :timed_out
+
+          if e == :no_retry
+            # Do not schedule the job to be retried
+            change_state(:failed)
+          elsif e.kind_of?(Exception)
+            # Handle exception and schedule for retry
+            Juggler.exception_handler.call(e)
+            change_state(:retried)
+          else
+            Juggler.logger.debug { "#{to_s}: failed with #{e.inspect}" }
+            change_state(:retried)
+          end
+        }
+        @strategy_deferrable = sd
+      rescue => e
+        Juggler.exception_handler.call(e)
+        change_state(:retried)
+      end
+    end
+    
+    def timeout_strategy
+      @strategy_deferrable.fail(:timed_out)
+    end
+
+    def backoff
+      Juggler.backoff_function.call(self, @stats)
+    end
+  end
+end

data/lib/juggler/runner.rb

@@ -1,90 +1,201 @@
 class Juggler
+  # Stopping: This is rather complex. The point of the __STOP__ malarkey it to 
+  # unblock a blocking reserve so that delete and release commands can be 
+  # actioned on the currently running jobs before shutdown. Also a 
+  # Juggler.shutdown_grace_timeout period is availble for jobs to complete before the 
+  # eventmachine is stopped
+  # 
   class Runner
     class << self
-      def start
-        @started ||= begin
-          Signal.trap('INT') { EM.stop }
-          Signal.trap('TERM') { EM.stop }
+      def start(runner)
+        @runners ||= []
+        @runners << runner
+        
+        @signals_setup ||= begin
+          %w{INT TERM}.each do |sig|
+            Signal.trap(sig) {
+              stop_all_runners_with_grace
+            }
+          end
           true
         end
       end
+      
+      private
+      
+      def stop_all_runners_with_grace
+        # Trigger each runner to shut down
+        @runners.each { |r| r.stop }
+        
+        Juggler.logger.info {
+          "Giving processes #{Juggler.shutdown_grace_timeout}s grace period to exit"
+        }
+        
+        EM::PeriodicTimer.new(0.1) {
+          if !@runners.any? { |r| r.running? }
+            Juggler.logger.info "Exited cleanly"
+            EM.stop
+          end
+        }
+        
+        EM::Timer.new(Juggler.shutdown_grace_timeout) do
+          Juggler.logger.info {
+            "Force exited after #{Juggler.shutdown_grace_timeout}s with tasks running"
+          }
+          EM.stop
+        end
+      end
     end
 
     def initialize(method, concurrency, strategy)
       @strategy = strategy
       @concurrency = concurrency
       @queue = method.to_s
+      
       @running = []
+      @reserved = false
+    end
+    
+    # We potentially need to issue a new reserve call after a job is reserved 
+    # (if we're not at the concurrency limit), and after a job completes 
+    # (unless we're already reserving)
+    def reserve_if_necessary
+      if @on && @connection.connected? && !@reserved && @running.size < @concurrency
+        Juggler.logger.debug "#{to_s}: Reserving"
+        reserve
+      end
     end
 
     def reserve
-      beanstalk_job = connection.reserve(0)
+      @reserved = true
+      
+      reserve_call = connection.reserve
+      
+      reserve_call.callback do |job|
+        @reserved = false
 
-      begin
-        params = Marshal.load(beanstalk_job.body)
-      rescue => e
-        handle_exception(e, "Exception unmarshaling #{@queue} job")
-        beanstalk_job.delete
-        return
-      end
+        begin
+          params = Marshal.load(job.body)
+        rescue => e
+          handle_exception(e, "#{to_s}: Exception unmarshaling #{@queue} job")
+          connection.delete(job)
+          next
+        end
+        
+        if params == "__STOP__"
+          connection.delete(job)
+          next
+        end
+        
+        job_runner = JobRunner.new(job, params, @strategy)
+        
+        @running << job_runner
 
-      begin
-        job = @strategy.call(params)
-      rescue => e
-        handle_exception(e, "Exception calling #{@queue} strategy")
-        beanstalk_job.decay
-        return
-      end
+        Juggler.logger.debug {
+          "#{to_s}: Excecuting #{@running.size} jobs"
+        }
 
-      EM::Timer.new(beanstalk_job.ttr - 2) do
-        job.fail(:timeout)
-      end
+        # We may reserve after job is running (after fetching stats)
+        job_runner.bind(:running) {
+          reserve_if_necessary
+        }
 
-      @running << job
-      job.callback do
-        @running.delete(job)
-        beanstalk_job.delete
+        # Also may reserve when a job is done
+        job_runner.bind(:done) {
+          @running.delete(job_runner)
+          reserve_if_necessary
+        }
+
+        job_runner.run
       end
-      job.errback do |error|
-        Juggler.logger.info("#{@queue} job failed: #{error}")
-        @running.delete(job)
-        # Built in exponential backoff
-        beanstalk_job.decay
+      
+      reserve_call.errback do |error|
+        @reserved = false
+        
+        if error == :deadline_soon
+          # This doesn't necessarily mean that a job has taken too long, it is 
+          # quite likely that the blocking reserve is just stopping jobs from 
+          # being deleted
+          Juggler.logger.debug "#{to_s}: Reserve terminated (deadline_soon)"
+
+          check_all_reserved_jobs.callback {
+            reserve_if_necessary
+          }
+        elsif error == :disconnected
+          Juggler.logger.warn "#{to_s}: Reserve terminated (beanstalkd disconnected)"
+        else
+          Juggler.logger.error "#{to_s}: Unexpected error: #{error}"
+          reserve_if_necessary
+        end
       end
-    rescue Beanstalk::TimedOut
-    rescue Beanstalk::NotConnected
-      Juggler.logger.fatal "Could not connect any beanstalk hosts. " \
-        "Retrying in 1s."
-      sleep 1
-    rescue => e
-      handle_exception(e, "Unhandled exception")
-      beanstalk_job.delete if beanstalk_job
     end
 
     def run
-      EM.add_periodic_timer do
-        reserve if spare_slot?
-      end
-      Runner.start
+      @on = true
+      Runner.start(self)
+      # Creates beanstalkd connection - reserve happens on connect
+      connection
     end
 
-    private
+    def stop
+      @on = false
 
-    def spare_slot?
-      @running.size < @concurrency
+      # See class documentation on stopping
+      if @reserved
+        Juggler.throw(@queue, "__STOP__")
+      end
+    end
+    
+    def running?
+      @running.size > 0
+    end
+    
+    def to_s
+      "Tube #{@queue}"
     end
 
+    private
+
     def handle_exception(e, message)
-      Juggler.logger.error "#{message}: #{e.class} #{e.message}"
+      Juggler.logger.error "#{message}: #{e.message} (#{e.class})"
       Juggler.logger.debug e.backtrace.join("\n")
     end
 
     def connection
-      @pool ||= begin
-        pool = Beanstalk::Pool.new(Juggler.hosts)
-        pool.watch(@queue)
-        pool
+      @connection ||= begin
+        c = EMJack::Connection.new({
+          :host => Juggler.server.host,
+          :port => Juggler.server.port,
+        })
+        c.on_connect {
+          c.watch(@queue)
+          reserve_if_necessary
+        }
+        c
+      end
+    end
+    
+    # Iterates over all jobs reserved on this connection and fails them if 
+    # they're within 1s of their timeout. Returns a callback which completes 
+    # when all jobs have been checked
+    def check_all_reserved_jobs
+      dd = EM::DefaultDeferrable.new
+      
+      @running.each do |job_runner|
+        job_runner.check_for_timeout
+      end
+      
+      # Wait 1s before reserving or we'll just get DEALINE_SOON again
+      # "If the client issues a reserve command during the safety margin, 
+      # <snip>, the server will respond with: DEADLINE_SOON"
+      #
+      # In theory, one should not need to do this since reserve will already
+      # be triggered as a callback on the job that has timed out
+      EM::Timer.new(1) do
+        dd.succeed
       end
+      
+      dd
     end
   end
 end

data/lib/juggler/state_machine.rb

@@ -0,0 +1,81 @@
+# Special eventmachine state machine
+#
+# Callbacks are defined for before :exit, :pre enter and after :enter
+# Asynchrous callbacks are not properly supported yet - only :pre must return
+# a deferrable which will continue the callback chain on complete
+# 
+# state :foobar, :enter => 'get_http'
+# 
+module Juggler::StateMachine
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+  
+  def state
+    @_state
+  end
+  
+  def change_state(new_state)
+    old_state = @_state
+    
+    Juggler.logger.debug "#{to_s}: Changing state: #{old_state} to #{new_state}"
+    
+    return nil if old_state == new_state
+    
+    raise "#{to_s}: Invalid state #{new_state}" unless self.class.states[new_state]
+
+    if method = self.class.states[new_state][:pre]
+      deferable = self.send(method)
+      deferable.callback {
+        run_synchronous_callbacks(old_state, new_state)
+      }
+      deferable.errback {
+        Juggler.logger.warn "#{to_s}: State change aborted - pre failed"
+      }
+    else
+      run_synchronous_callbacks(old_state, new_state)
+    end
+
+    return true
+  end
+
+  def bind(state, &callback)
+    @on_state ||= Hash.new { |h, k| h[k] = [] }
+    @on_state[state] << callback
+  end
+
+  private
+
+  def run_synchronous_callbacks(old_state, new_state)
+    catch :halt do
+      if callbacks = self.class.states[old_state][:exit]
+        [callbacks].flatten.each { |c| self.send(c) }
+      end
+
+      set_state(new_state)
+
+      if callbacks = self.class.states[new_state][:enter]
+        [callbacks].flatten.each { |c| self.send(c) }
+      end
+    end
+  end
+
+  def set_state(new_state)
+    @_state = new_state
+
+    if @on_state && (callbacks = @on_state[new_state])
+      callbacks.each { |c| c.call(self) }
+    end
+  end
+  
+  module ClassMethods
+    def states
+      @_states
+    end
+  
+    def state(name, callbacks = {})
+      @_states ||= {}
+      @_states[name] = callbacks
+    end
+  end
+end

data/spec/integration/juggler_spec.rb

@@ -0,0 +1,100 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe "Juggler" do
+  include EM::SpecHelper
+  
+  before :each do
+    # Reset state
+    Juggler.instance_variable_set(:@connection, nil)
+    Juggler::Runner.instance_variable_set(:@runners, nil)
+    
+    # Start clean beanstalk instance for each test
+    Juggler.server = "beanstalk://localhost:10001"
+    system "beanstalkd -p 10001 &"
+    sleep 0.1
+  end
+  
+  after :each do
+    # TODO: Use pid
+    system "killall beanstalkd"
+  end
+  
+  it "should successfully excecute one job" do
+    em(1) do
+      params_for_jobs_received = []
+      Juggler.juggle(:some_task, 1) { |df, params|
+        params_for_jobs_received << params
+        df.succeed_later_with(nil)
+      }
+      Juggler.throw(:some_task, {:some => "params"})
+      
+      EM.add_timer(0.1) {
+        params_for_jobs_received.should == [{:some => "params"}]
+        done
+      }
+    end
+  end
+  
+  it "should run correct number of jobs concurrently" do
+    em(1) do
+      params_for_jobs_received = []
+      Juggler.juggle(:some_task, 2) { |df, params|
+        params_for_jobs_received << params
+        df.succeed_later_with(nil, 0.2)
+      }
+      
+      10.times { |i| Juggler.throw(:some_task, i) }
+      
+      EM.add_timer(0.3) {
+        # After 0.3 seconds, 2 jobs should have completed, and 2 more started
+        params_for_jobs_received.should == [0, 1, 2, 3]
+        done
+      }
+    end
+  end
+  
+  it "should stop em after jobs completed when signaled to QUIT" do
+    job_finished = false
+    job_started = false
+    em(1) do
+      Juggler.juggle(:some_task, 1) { |df, params|
+        job_started = true
+        EM.add_timer(0.2) {
+          job_finished = true
+          df.succeed
+        }
+      }
+      Juggler.throw(:some_task, 'foo')
+      EM.add_timer(0.1) {
+        job_started.should == true
+        job_finished.should == false
+        Juggler::Runner.send(:stop_all_runners_with_grace)
+      }
+    end
+    job_finished.should == true
+  end
+  
+  it "should kill jobs that do not complete within shutdown_grace_timeout" do
+    job_finished = false
+    job_started = false
+    em(1) do
+      Juggler.shutdown_grace_timeout = 0.1
+      Juggler.juggle(:some_task, 1) { |params|
+        dd = EM::DefaultDeferrable.new
+        job_started = true
+        EM.add_timer(1) {
+          job_finished = true
+          dd.succeed
+        }
+        dd
+      }
+      Juggler.throw(:some_task, 'foo')
+      EM.add_timer(0.1) {
+        job_started.should == true
+        job_finished.should == false
+        Juggler::Runner.send(:stop_all_runners_with_grace)
+      }
+    end
+    job_finished.should == false
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,53 @@
 $:.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 
 require 'juggler'
+
+Juggler.logger.level = Logger::FATAL
+
+require 'em-spec/rspec'
+
+module EM
+  class ChainableDeferrable
+    include Deferrable
+
+    def callback(&block)
+      super
+      self
+    end
+
+    def errback(&block)
+      super
+      self
+    end
+  end
+
+  module Deferrable
+    def succeed_later_with(callback, time = 0)
+      EM.add_timer(time) {
+        succeed(callback)
+      }
+    end
+
+    def fail_later_with(callback, time = 0)
+      EM.add_timer(time) {
+        fail(callback)
+      }
+    end
+  end
+end
+
+def stub_deferrable(callback, time = 0)
+  d = EM::ChainableDeferrable.new
+  EM.add_timer(time) {
+    d.succeed(callback)
+  }
+  d
+end
+
+def stub_failing_deferrable(callback, time = 0)
+  d = EM::ChainableDeferrable.new
+  EM.add_timer(time) {
+    d.fail(callback)
+  }
+  d
+end

data/spec/unit/job_runner_spec.rb

@@ -0,0 +1,192 @@
+require File.expand_path('../../spec_helper', __FILE__)
+
+# This spec describes the behaviour of job runner by mocking the interface to 
+# beanstalk as exposed by em-jack
+# 
+describe Juggler::JobRunner do
+  include EM::SpecHelper
+  
+  it "should run job and delete in the success case" do
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1, 
+        :stats => stub_deferrable({"time-left" => 2})
+      })
+      
+      job.should_receive(:delete).and_return(stub_deferrable(nil))
+      
+      strategy = lambda { |df, params|
+        df.succeed_later_with(nil, 0.2)
+      }
+      
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+      
+      # To check that check_for_timeout doesn't timeout in the sucess case
+      EM.add_timer(0.1) {
+        jobrunner.check_for_timeout
+      }
+      
+      jobrunner.bind(:succeeded) {
+        @state = :succeeded
+      }
+      
+      jobrunner.bind(:done) { 
+        @state.should == :succeeded
+        done
+      }
+    }
+  end
+  
+  it "should release job for retry if exception calling strategy" do
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1, 
+        :stats => stub_deferrable({"time-left" => 2, "delay" => 0})
+      })
+      
+      job.should_receive(:release).with({:delay => 1}).
+        and_return(stub_deferrable(nil))
+      
+      strategy = lambda { |df, params|
+        raise 'strategy blows up'
+      }
+      
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+      
+      jobrunner.bind(:retried) {
+        @state = :retried
+      }
+      
+      jobrunner.bind(:done) { 
+        @state.should == :retried
+        done
+      }
+    }
+  end
+  
+  it "should release job for retry and call exception handler if job deferrable fails with an exception" do
+    asserts = 0
+
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1,
+        :stats => stub_deferrable({"time-left" => 2, "delay" => 0})
+      })
+
+      job.should_receive(:release).with({:delay => 1}).
+        and_return(stub_deferrable(nil))
+
+      Juggler.exception_handler = lambda { |e|
+        e.message.should == "FAIL"
+        asserts += 1
+      }
+
+      strategy = lambda { |df, params|
+        EM.next_tick {
+          df.fail(RuntimeError.new("FAIL"))
+        }
+      }
+
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+
+      jobrunner.bind(:retried) {
+        asserts += 1
+      }
+
+      jobrunner.bind(:done) {
+        asserts.should == 2
+        done
+      }
+    }
+  end
+
+  it "should release job for retry if job fails with no arguments" do
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1, 
+        :stats => stub_deferrable({"time-left" => 2, "delay" => 0})
+      })
+      
+      job.should_receive(:release).with({:delay => 1}).
+        and_return(stub_deferrable(nil))
+      
+      strategy = lambda { |df, params|
+        df.fail_later_with(nil)
+      }
+      
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+      
+      jobrunner.bind(:retried) {
+        @state = :retried
+      }
+      
+      jobrunner.bind(:done) { 
+        @state.should == :retried
+        done
+      }
+    }
+  end
+  
+  it "should fail and delete job if job fails with :no_retry" do
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1, 
+        :stats => stub_deferrable({"time-left" => 2, "delay" => 0})
+      })
+      
+      job.should_receive(:delete).and_return(stub_deferrable(nil))
+      
+      strategy = lambda { |df, params|
+        df.fail_later_with(:no_retry)
+      }
+      
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+      
+      jobrunner.bind(:failed) {
+        @state = :failed
+      }
+      
+      jobrunner.bind(:done) { 
+        @state.should == :failed
+        done
+      }
+    }
+  end
+  
+  it "should retry job if strategy deferrable exceeds timeout" do
+    em(1) {
+      job = mock(:job, {
+        :jobid => 1, 
+        :stats => stub_deferrable({"time-left" => 1, "delay" => 0})
+      })
+      
+      job.should_receive(:release).with({:delay => 1}).
+        and_return(stub_deferrable(nil))
+      
+      strategy = lambda { |df, params|
+        df.succeed_later_with(nil, 2)
+      }
+      
+      jobrunner = Juggler::JobRunner.new(job, {}, strategy)
+      jobrunner.run
+      
+      EM.add_timer(0.1) {
+        jobrunner.check_for_timeout
+      }
+      
+      jobrunner.bind(:timed_out) {
+        @state = :timed_out
+      }
+      
+      jobrunner.bind(:done) { 
+        @state.should == :timed_out
+        done
+      }
+    }
+  end
+end

metadata

@@ -1,71 +1,99 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: juggler
-version: !ruby/object:Gem::Version 
-  version: 0.0.2
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Martyn Loughran
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2009-10-21 00:00:00 +01:00
-default_executable: 
-dependencies: []
-
+date: 2012-08-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: em-jack
+  requirement: &70241134047560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70241134047560
+- !ruby/object:Gem::Dependency
+  name: em-spec
+  requirement: &70241134047120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70241134047120
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70241134046660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70241134046660
 description: Juggling background jobs with EventMachine and Beanstalkd
-email: me@mloughran.com
+email:
+- me@mloughran.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
-- LICENSE
-- README.md
-files: 
+extra_rdoc_files: []
+files:
 - .gitignore
+- Gemfile
 - LICENSE
 - README.md
 - Rakefile
 - VERSION
+- examples/app.rb
 - examples/test_consumer.rb
 - examples/test_producer.rb
+- juggler.gemspec
 - lib/juggler.rb
+- lib/juggler/job_runner.rb
 - lib/juggler/runner.rb
-- spec/juggler_spec.rb
+- lib/juggler/state_machine.rb
+- spec/integration/juggler_spec.rb
 - spec/spec_helper.rb
-has_rdoc: true
-homepage: http://github.com/mloughran/juggler
+- spec/unit/job_runner_spec.rb
+homepage: https://github.com/mloughran/juggler
 licenses: []
-
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
-require_paths: 
+rdoc_options: []
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Juggling background jobs with EventMachine and Beanstalkd
-test_files: 
-- spec/juggler_spec.rb
-- spec/runner_spec.rb
+test_files:
+- spec/integration/juggler_spec.rb
 - spec/spec_helper.rb
-- examples/app.rb
-- examples/test_consumer.rb
-- examples/test_producer.rb
+- spec/unit/job_runner_spec.rb
+has_rdoc: 

data/spec/juggler_spec.rb

@@ -1,7 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe Juggler do
-  it "should put jobs on queue" do
-    Juggler.throw('method', {:foo => "bar"})
-  end
-end

data/spec/runner_spec.rb

@@ -1,21 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe Juggler::Runner do
-  it "should delete job from beanstalkd when successful" do
-    @mock_beanstalk = mock(Beanstalk::Pool)
-    @mock_job = mock(Beanstalk::Job, :body => "foo")
-    Beanstalk::Pool.should_receive(:new).and_return(@mock_beanstalk)
-    
-    @mock_job.should_receive(:delete)
-    
-    Juggler::Runner.new(:method, 1, lambda do
-      deferrable = EM::DefaultDeferrable.new
-      deferrable.set_deferred_status :succeeded, nil
-      deferrable
-    end).run
-    
-    sleep 1
-  end
-  
-  it "should retry job if not successful"
-end