delayed_job 1.8.5 → 1.9.0pre

data/lib/delayed/command.rb

@@ -36,6 +36,8 @@ module Delayed
     end
   
     def daemonize
+      Delayed::Worker.backend.before_fork
+
       ObjectSpace.each_object(File) do |file|
         @files_to_reopen << file unless file.closed?
       end
@@ -64,11 +66,11 @@ module Delayed
       if Delayed::Worker.logger.respond_to? :auto_flushing=
         Delayed::Worker.logger.auto_flushing = true
       end
-      ActiveRecord::Base.connection.reconnect!
-      
-      Delayed::Job.worker_name = "#{worker_name} #{Delayed::Job.worker_name}"
+      Delayed::Worker.backend.after_fork
       
-      Delayed::Worker.new(@options).start  
+      worker = Delayed::Worker.new(@options)
+      worker.name_prefix = "#{worker_name} "
+      worker.start
     rescue => e
       Rails.logger.fatal e
       STDERR.puts e.message

data/lib/delayed/performable_method.rb

@@ -1,7 +1,19 @@
+class Class
+  def load_for_delayed_job(arg)
+    self
+  end
+  
+  def dump_for_delayed_job
+    name
+  end
+end
+
 module Delayed
   class PerformableMethod < Struct.new(:object, :method, :args)
-    CLASS_STRING_FORMAT = /^CLASS\:([A-Z][\w\:]+)$/
-    AR_STRING_FORMAT    = /^AR\:([A-Z][\w\:]+)\:(\d+)$/
+    STRING_FORMAT = /^LOAD\;([A-Z][\w\:]+)(?:\;(\w+))?$/
+    
+    class LoadError < StandardError
+    end
 
     def initialize(object, method, args)
       raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method)
@@ -11,45 +23,40 @@ module Delayed
       self.method = method.to_sym
     end
     
-    def display_name  
-      case self.object
-      when CLASS_STRING_FORMAT then "#{$1}.#{method}"
-      when AR_STRING_FORMAT    then "#{$1}##{method}"
-      else "Unknown##{method}"
-      end      
-    end    
-
+    def display_name
+      if STRING_FORMAT === object
+        "#{$1}#{$2 ? '#' : '.'}#{method}"
+      else
+        "#{object.class}##{method}"
+      end
+    end
+    
     def perform
       load(object).send(method, *args.map{|a| load(a)})
-    rescue ActiveRecord::RecordNotFound
-      # We cannot do anything about objects which were deleted in the meantime
+    rescue PerformableMethod::LoadError
+      # We cannot do anything about objects that can't be loaded
       true
     end
 
     private
 
-    def load(arg)
-      case arg
-      when CLASS_STRING_FORMAT then $1.constantize
-      when AR_STRING_FORMAT    then $1.constantize.find($2)
-      else arg
+    def load(obj)
+      if STRING_FORMAT === obj
+        $1.constantize.load_for_delayed_job($2)
+      else
+        obj
       end
+    rescue => e
+      Delayed::Worker.logger.warn "Could not load object for job: #{e.message}"
+      raise PerformableMethod::LoadError
     end
 
-    def dump(arg)
-      case arg
-      when Class              then class_to_string(arg)
-      when ActiveRecord::Base then ar_to_string(arg)
-      else arg
+    def dump(obj)
+      if obj.respond_to?(:dump_for_delayed_job)
+        "LOAD;#{obj.dump_for_delayed_job}"
+      else
+        obj
       end
     end
-
-    def ar_to_string(obj)
-      "AR:#{obj.class}:#{obj.id}"
-    end
-
-    def class_to_string(obj)
-      "CLASS:#{obj.name}"
-    end
   end
-end
+end

data/lib/delayed/worker.rb

@@ -1,24 +1,61 @@
+require 'timeout'
+require 'active_support/core_ext/numeric/time'
+
 module Delayed
   class Worker
-    @@sleep_delay = 5
+    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :sleep_delay, :logger
+    self.sleep_delay = 5
+    self.max_attempts = 25
+    self.max_run_time = 4.hours
+    
+    # By default failed jobs are destroyed after too many attempts. If you want to keep them around
+    # (perhaps to inspect the reason for the failure), set this to false.
+    cattr_accessor :destroy_failed_jobs
+    self.destroy_failed_jobs = true
     
-    cattr_accessor :sleep_delay
-
-    cattr_accessor :logger
     self.logger = if defined?(Merb::Logger)
       Merb.logger
     elsif defined?(RAILS_DEFAULT_LOGGER)
       RAILS_DEFAULT_LOGGER
     end
 
+    # name_prefix is ignored if name is set directly
+    attr_accessor :name_prefix
+    
+    cattr_reader :backend
+    
+    def self.backend=(backend)
+      if backend.is_a? Symbol
+        require "delayed/backend/#{backend}"
+        backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
+      end
+      @@backend = backend
+      silence_warnings { ::Delayed.const_set(:Job, backend) }
+    end
+
     def initialize(options={})
       @quiet = options[:quiet]
-      Delayed::Job.min_priority = options[:min_priority] if options.has_key?(:min_priority)
-      Delayed::Job.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+      self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
+      self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+    end
+
+    # Every worker has a unique name which by default is the pid of the process. There are some
+    # advantages to overriding this with something which survives worker retarts:  Workers can#
+    # safely resume working on tasks which are locked by themselves. The worker will assume that
+    # it crashed before.
+    def name
+      return @name unless @name.nil?
+      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
+    end
+
+    # Sets the name of the worker.
+    # Setting the name to nil will reset the default worker name
+    def name=(val)
+      @name = val
     end
 
     def start
-      say "*** Starting job worker #{Delayed::Job.worker_name}"
+      say "*** Starting job worker #{name}"
 
       trap('TERM') { say 'Exiting...'; $exit = true }
       trap('INT')  { say 'Exiting...'; $exit = true }
@@ -27,7 +64,7 @@ module Delayed
         result = nil
 
         realtime = Benchmark.realtime do
-          result = Delayed::Job.work_off
+          result = work_off
         end
 
         count = result.sum
@@ -44,13 +81,86 @@ module Delayed
       end
 
     ensure
-      Delayed::Job.clear_locks!
+      Delayed::Job.clear_locks!(name)
+    end
+    
+    # Do num jobs and return stats on success/failure.
+    # Exit early if interrupted.
+    def work_off(num = 100)
+      success, failure = 0, 0
+
+      num.times do
+        case reserve_and_run_one_job
+        when true
+            success += 1
+        when false
+            failure += 1
+        else
+          break  # leave if no work could be done
+        end
+        break if $exit # leave if we're exiting
+      end
+
+      return [success, failure]
+    end
+    
+    def run(job)
+      runtime =  Benchmark.realtime do
+        Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
+        job.destroy
+      end
+      # TODO: warn if runtime > max_run_time ?
+      say "* [JOB] #{name} completed after %.4f" % runtime
+      return true  # did work
+    rescue Exception => e
+      handle_failed_job(job, e)
+      return false  # work failed
+    end
+    
+    # Reschedule the job in the future (when a job fails).
+    # Uses an exponential scale depending on the number of failed attempts.
+    def reschedule(job, time = nil)
+      if (job.attempts += 1) < self.class.max_attempts
+        time ||= Job.db_time_now + (job.attempts ** 4) + 5
+        job.run_at = time
+        job.unlock
+        job.save!
+      else
+        say "* [JOB] PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+        self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
+      end
     end
 
-    def say(text)
+    def say(text, level = Logger::INFO)
       puts text unless @quiet
-      logger.info text if logger
+      logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
     end
 
+  protected
+    
+    def handle_failed_job(job, error)
+      job.last_error = error.message + "\n" + error.backtrace.join("\n")
+      say "* [JOB] #{name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
+      reschedule(job)
+    end
+    
+    # Run the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def reserve_and_run_one_job
+
+      # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+      # this leads to a more even distribution of jobs across the worker processes
+      job = Delayed::Job.find_available(name, 5, self.class.max_run_time).detect do |job|
+        if job.lock_exclusively!(self.class.max_run_time, name)
+          say "* [Worker(#{name})] acquired lock on #{job.name}"
+          true
+        else
+          say "* [Worker(#{name})] failed to acquire exclusive lock for #{job.name}", Logger::WARN
+          false
+        end
+      end
+
+      run(job) if job
+    end
   end
 end

data/lib/delayed_job.rb

@@ -1,8 +1,8 @@
-autoload :ActiveRecord, 'activerecord'
+require 'active_support'
 
 require File.dirname(__FILE__) + '/delayed/message_sending'
 require File.dirname(__FILE__) + '/delayed/performable_method'
-require File.dirname(__FILE__) + '/delayed/job'
+require File.dirname(__FILE__) + '/delayed/backend/base'
 require File.dirname(__FILE__) + '/delayed/worker'
 
 Object.send(:include, Delayed::MessageSending)   

data/rails/init.rb

@@ -0,0 +1,12 @@
+require 'delayed_job'
+
+config.after_initialize do
+  Delayed::Worker.backend ||= if defined?(ActiveRecord)
+    :active_record
+  elsif defined?(MongoMapper)
+    :mongo_mapper
+  else
+    Delayed::Worker.logger.warn "Could not decide on a backend, defaulting to active_record"
+    :active_record
+  end
+end

data/spec/backend/active_record_job_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require 'delayed/backend/active_record'
+
+describe Delayed::Job do
+  before(:all) do
+    @backend = Delayed::Job
+  end
+  
+  before(:each) do
+    Delayed::Worker.max_priority = nil
+    Delayed::Worker.min_priority = nil
+    Delayed::Job.delete_all
+    SimpleJob.runs = 0
+  end
+  
+  after do
+    Time.zone = nil
+  end
+  
+  it_should_behave_like 'a backend'
+
+  context "db_time_now" do
+    it "should return time in current time zone if set" do
+      Time.zone = 'Eastern Time (US & Canada)'
+      %w(EST EDT).should include(Delayed::Job.db_time_now.zone)
+    end
+    
+    it "should return UTC time if that is the AR default" do
+      Time.zone = nil
+      ActiveRecord::Base.default_timezone = :utc
+      Delayed::Job.db_time_now.zone.should == 'UTC'
+    end
+
+    it "should return local time if that is the AR default" do
+      Time.zone = 'Central Time (US & Canada)'
+      ActiveRecord::Base.default_timezone = :local
+      %w(CST CDT).should include(Delayed::Job.db_time_now.zone)
+    end
+  end
+  
+end

data/spec/backend/mongo_mapper_job_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+
+require 'delayed/backend/mongo_mapper'
+
+describe Delayed::Backend::MongoMapper::Job do
+  before(:all) do
+    @backend = Delayed::Backend::MongoMapper::Job
+  end
+  
+  before(:each) do
+    MongoMapper.database.collections.each(&:remove)
+  end
+  
+  it_should_behave_like 'a backend'
+
+  describe "indexes" do
+    it "should have combo index on priority and run_at" do
+      @backend.collection.index_information.detect { |index| index[0] == 'priority_1_run_at_1' }.should_not be_nil
+    end
+
+    it "should have index on locked_by" do
+      @backend.collection.index_information.detect { |index| index[0] == 'locked_by_1' }.should_not be_nil
+    end
+  end
+  
+  describe "delayed method" do
+    class MongoStoryReader
+      def read(story)
+        "Epilog: #{story.tell}"
+      end
+    end
+    
+    class MongoStory
+      include ::MongoMapper::Document
+      key :text, String
+      
+      def tell
+        text
+      end
+    end
+    
+    it "should ignore not found errors because they are permanent" do
+      story = MongoStory.create :text => 'Once upon a time...'
+      job = story.send_later(:tell)
+      story.destroy
+      lambda { job.invoke_job }.should_not raise_error
+    end
+
+    it "should store the object as string" do
+      story = MongoStory.create :text => 'Once upon a time...'
+      job = story.send_later(:tell)
+
+      job.payload_object.class.should   == Delayed::PerformableMethod
+      job.payload_object.object.should  == "LOAD;MongoStory;#{story.id}"
+      job.payload_object.method.should  == :tell
+      job.payload_object.args.should    == []
+      job.payload_object.perform.should == 'Once upon a time...'
+    end
+
+    it "should store arguments as string" do
+      story = MongoStory.create :text => 'Once upon a time...'
+      job = MongoStoryReader.new.send_later(:read, story)
+      job.payload_object.class.should   == Delayed::PerformableMethod
+      job.payload_object.method.should  == :read
+      job.payload_object.args.should    == ["LOAD;MongoStory;#{story.id}"]
+      job.payload_object.perform.should == 'Epilog: Once upon a time...'
+    end
+  end
+end

data/spec/backend/shared_backend_spec.rb

@@ -0,0 +1,244 @@
+shared_examples_for 'a backend' do
+  def create_job(opts = {})
+    @backend.create(opts.merge(:payload_object => SimpleJob.new))
+  end
+
+  before do
+    SimpleJob.runs = 0
+  end
+  
+  it "should set run_at automatically if not set" do
+    @backend.create(:payload_object => ErrorJob.new ).run_at.should_not be_nil
+  end
+
+  it "should not set run_at automatically if already set" do
+    later = @backend.db_time_now + 5.minutes
+    @backend.create(:payload_object => ErrorJob.new, :run_at => later).run_at.should be_close(later, 1)
+  end
+
+  it "should raise ArgumentError when handler doesn't respond_to :perform" do
+    lambda { @backend.enqueue(Object.new) }.should raise_error(ArgumentError)
+  end
+
+  it "should increase count after enqueuing items" do
+    @backend.enqueue SimpleJob.new
+    @backend.count.should == 1
+  end
+  
+  it "should be able to set priority when enqueuing items" do
+    @job = @backend.enqueue SimpleJob.new, 5
+    @job.priority.should == 5
+  end
+
+  it "should be able to set run_at when enqueuing items" do
+    later = @backend.db_time_now + 5.minutes
+    @job = @backend.enqueue SimpleJob.new, 5, later
+    @job.run_at.should be_close(later, 1)
+  end
+
+  it "should work with jobs in modules" do
+    M::ModuleJob.runs = 0
+    job = @backend.enqueue M::ModuleJob.new
+    lambda { job.invoke_job }.should change { M::ModuleJob.runs }.from(0).to(1)
+  end
+                   
+  it "should raise an DeserializationError when the job class is totally unknown" do
+    job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
+    lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
+  end
+
+  it "should try to load the class when it is unknown at the time of the deserialization" do
+    job = @backend.new :handler => "--- !ruby/object:JobThatDoesNotExist {}"
+    job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
+    lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
+  end
+
+  it "should try include the namespace when loading unknown objects" do
+    job = @backend.new :handler => "--- !ruby/object:Delayed::JobThatDoesNotExist {}"
+    job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
+    lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
+  end
+
+  it "should also try to load structs when they are unknown (raises TypeError)" do
+    job = @backend.new :handler => "--- !ruby/struct:JobThatDoesNotExist {}"
+    job.should_receive(:attempt_to_load).with('JobThatDoesNotExist').and_return(true)
+    lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
+  end
+
+  it "should try include the namespace when loading unknown structs" do
+    job = @backend.new :handler => "--- !ruby/struct:Delayed::JobThatDoesNotExist {}"
+    job.should_receive(:attempt_to_load).with('Delayed::JobThatDoesNotExist').and_return(true)
+    lambda { job.payload_object.perform }.should raise_error(Delayed::Backend::DeserializationError)
+  end
+  
+  describe "find_available" do
+    it "should not find failed jobs" do
+      @job = create_job :attempts => 50, :failed_at => @backend.db_time_now
+      @backend.find_available('worker', 5, 1.second).should_not include(@job)
+    end
+    
+    it "should not find jobs scheduled for the future" do
+      @job = create_job :run_at => (@backend.db_time_now + 1.minute)
+      @backend.find_available('worker', 5, 4.hours).should_not include(@job)
+    end
+    
+    it "should not find jobs locked by another worker" do
+      @job = create_job(:locked_by => 'other_worker', :locked_at => @backend.db_time_now - 1.minute)
+      @backend.find_available('worker', 5, 4.hours).should_not include(@job)
+    end
+    
+    it "should find open jobs" do
+      @job = create_job
+      @backend.find_available('worker', 5, 4.hours).should include(@job)
+    end
+    
+    it "should find expired jobs" do
+      @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now - 2.minutes)
+      @backend.find_available('worker', 5, 1.minute).should include(@job)
+    end
+    
+    it "should find own jobs" do
+      @job = create_job(:locked_by => 'worker', :locked_at => (@backend.db_time_now - 1.minutes))
+      @backend.find_available('worker', 5, 4.hours).should include(@job)
+    end
+
+    it "should find only the right amount of jobs" do
+      10.times { create_job }
+      @backend.find_available('worker', 7, 4.hours).should have(7).jobs
+    end
+  end
+  
+  context "when another worker is already performing an task, it" do
+
+    before :each do
+      @job = @backend.create :payload_object => SimpleJob.new, :locked_by => 'worker1', :locked_at => @backend.db_time_now - 5.minutes
+    end
+
+    it "should not allow a second worker to get exclusive access" do
+      @job.lock_exclusively!(4.hours, 'worker2').should == false
+    end
+
+    it "should allow a second worker to get exclusive access if the timeout has passed" do
+      @job.lock_exclusively!(1.minute, 'worker2').should == true
+    end      
+    
+    it "should be able to get access to the task if it was started more then max_age ago" do
+      @job.locked_at = 5.hours.ago
+      @job.save
+
+      @job.lock_exclusively! 4.hours, 'worker2'
+      @job.reload
+      @job.locked_by.should == 'worker2'
+      @job.locked_at.should > 1.minute.ago
+    end
+
+    it "should not be found by another worker" do
+      @backend.find_available('worker2', 1, 6.minutes).length.should == 0
+    end
+
+    it "should be found by another worker if the time has expired" do
+      @backend.find_available('worker2', 1, 4.minutes).length.should == 1
+    end
+
+    it "should be able to get exclusive access again when the worker name is the same" do
+      @job.lock_exclusively!(5.minutes, 'worker1').should be_true
+      @job.lock_exclusively!(5.minutes, 'worker1').should be_true
+      @job.lock_exclusively!(5.minutes, 'worker1').should be_true
+    end                                        
+  end
+  
+  context "when another worker has worked on a task since the job was found to be available, it" do
+
+    before :each do
+      @job = @backend.create :payload_object => SimpleJob.new
+      @job_copy_for_worker_2 = @backend.find(@job.id)
+    end
+
+    it "should not allow a second worker to get exclusive access if already successfully processed by worker1" do
+      @job.destroy
+      @job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
+    end
+
+    it "should not allow a second worker to get exclusive access if failed to be processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
+      @job.update_attributes(:attempts => 1, :run_at => 1.day.from_now)
+      @job_copy_for_worker_2.lock_exclusively!(4.hours, 'worker2').should == false
+    end
+  end
+
+  context "#name" do
+    it "should be the class name of the job that was enqueued" do
+      @backend.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
+    end
+
+    it "should be the method that will be called if its a performable method object" do
+      @job = Story.send_later(:create)
+      @job.name.should == "Story.create"
+    end
+
+    it "should be the instance method that will be called if its a performable method object" do
+      @job = Story.create(:text => "...").send_later(:save)
+      @job.name.should == 'Story#save'
+    end
+  end
+  
+  context "worker prioritization" do
+    before(:each) do
+      Delayed::Worker.max_priority = nil
+      Delayed::Worker.min_priority = nil
+    end
+
+    it "should fetch jobs ordered by priority" do
+      10.times { @backend.enqueue SimpleJob.new, rand(10) }
+      jobs = @backend.find_available('worker', 10)
+      jobs.size.should == 10
+      jobs.each_cons(2) do |a, b| 
+        a.priority.should <= b.priority
+      end
+    end
+
+    it "should only find jobs greater than or equal to min priority" do
+      min = 5
+      Delayed::Worker.min_priority = min
+      10.times {|i| @backend.enqueue SimpleJob.new, i }
+      jobs = @backend.find_available('worker', 10)
+      jobs.each {|job| job.priority.should >= min}
+    end
+
+    it "should only find jobs less than or equal to max priority" do
+      max = 5
+      Delayed::Worker.max_priority = max
+      10.times {|i| @backend.enqueue SimpleJob.new, i }
+      jobs = @backend.find_available('worker', 10)
+      jobs.each {|job| job.priority.should <= max}
+    end
+  end
+  
+  context "clear_locks!" do
+    before do
+      @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
+    end
+    
+    it "should clear locks for the given worker" do
+      @backend.clear_locks!('worker')
+      @backend.find_available('worker2', 5, 1.minute).should include(@job)
+    end
+    
+    it "should not clear locks for other workers" do
+      @backend.clear_locks!('worker1')
+      @backend.find_available('worker1', 5, 1.minute).should_not include(@job)
+    end
+  end
+  
+  context "unlock" do
+    before do
+      @job = create_job(:locked_by => 'worker', :locked_at => @backend.db_time_now)
+    end
+
+    it "should clear locks" do
+      @job.unlock
+      @job.locked_by.should be_nil
+      @job.locked_at.should be_nil
+    end
+  end
+  
+end