delayed_job 2.0.4 → 2.0.5

data/CHANGELOG

@@ -1,3 +1,10 @@
+2.0.5 - 2010-12-01
+* Added #reschedule_at hook on payload to determine when the job should be rescheduled [backported from 2.1]
+* Added --sleep-delay command line option [backported from 2.1]
+* Added 'delayed_job_server_role' Capistrano variable to allow delayed_job to run on its own worker server
+    set :delayed_job_server_role, :worker
+* Changed AR backend to reserve jobs using an UPDATE query to reduce worker contention [backported from 2.1]
+
 2.0.4 - 2010-11-14
 * Fix issue where dirty tracking prevented job from being properly unlocked
 * Add delayed_job_args variable for Capistrano recipe to allow configuration of started workers (e.g. "-n 2 --max-priority 10")

data/README.textile

@@ -19,13 +19,14 @@ This version is for Rails 2.x only.  For rails 3 support, install delayed_job 2.
 To install as a gem, add the following to @config/environment.rb@:
 
 <pre>
-config.gem 'delayed_job'
+config.gem 'delayed_job', :version => '~>2.0.4'
 </pre>
 
 Rake tasks are not automatically loaded from gems, so you'll need to add the following to your Rakefile:
 
 <pre>
 begin
+  gem 'delayed_job', '~>2.0.4'
   require 'delayed/tasks'
 rescue LoadError
   STDERR.puts "Run `rake gems:install` to install delayed_job"
@@ -35,7 +36,7 @@ end
 To install as a plugin:
 
 <pre>
-script/plugin install git://github.com/collectiveidea/delayed_job.git
+script/plugin install git://github.com/collectiveidea/delayed_job.git -r v2.0
 </pre>
 
 After delayed_job is installed, you will need to setup the backend.

data/VERSION

@@ -1 +1 @@
-2.0.4
+2.0.5

data/delayed_job.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{delayed_job}
-  s.version = "2.0.4"
+  s.version = "2.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brandon Keepers", "Tobias L\303\274tke"]
-  s.date = %q{2010-11-14}
+  s.date = %q{2010-12-01}
   s.description = %q{Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks.
 
 This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job).}

data/generators/delayed_job/templates/migration.rb

@@ -13,6 +13,7 @@ class CreateDelayedJobs < ActiveRecord::Migration
     end
 	
     add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+    add_index :delayed_jobs, :locked_by, :name => 'delayed_jobs_locked_by'
   end
   
   def self.down

data/lib/delayed/backend/active_record.rb

@@ -30,6 +30,10 @@ module Delayed
         }
         named_scope :by_priority, :order => 'priority ASC, run_at ASC'
         
+        named_scope :locked_by_worker, lambda{|worker_name, max_run_time|
+          {:conditions => ['locked_by = ? AND locked_at > ?', worker_name, db_time_now - max_run_time]}
+        }
+                
         def self.after_fork
           ::ActiveRecord::Base.connection.reconnect!
         end
@@ -38,15 +42,33 @@ module Delayed
         def self.clear_locks!(worker_name)
           update_all("locked_by = null, locked_at = null", ["locked_by = ?", worker_name])
         end
-
-        # Find a few candidate jobs to run (in case some immediately get locked by others).
-        def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
+        
+        def self.jobs_available_to_worker(worker_name, max_run_time)
           scope = self.ready_to_run(worker_name, max_run_time)
           scope = scope.scoped(:conditions => ['priority >= ?', Worker.min_priority]) if Worker.min_priority
           scope = scope.scoped(:conditions => ['priority <= ?', Worker.max_priority]) if Worker.max_priority
-      
+          scope.by_priority
+        end
+                
+        # Reserve a single job in a single update query.  This causes workers to serialize on the 
+        # database and avoids contention.
+        def self.reserve(worker, max_run_time = Worker.max_run_time)
+          affected_rows = 0
+          ::ActiveRecord::Base.silence do
+            affected_rows = update_all(["locked_at = ?, locked_by = ?", db_time_now, worker.name], jobs_available_to_worker(worker.name, max_run_time).scope(:find)[:conditions], :limit => 1)
+          end
+          
+          if affected_rows == 1
+            locked_by_worker(worker.name, max_run_time).first
+          else
+            nil
+          end
+        end
+        
+        # Find a few candidate jobs to run (in case some immediately get locked by others).
+        def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
           ::ActiveRecord::Base.silence do
-            scope.by_priority.all(:limit => limit)
+            jobs_available_to_worker(worker_name, max_run_time).all(:limit => limit)
           end
         end
 

data/lib/delayed/backend/base.rb

@@ -20,7 +20,15 @@ module Delayed
           run_at   = args[1]
           self.create(:payload_object => object, :priority => priority.to_i, :run_at => run_at)
         end
-        
+
+        def reserve(worker, max_run_time = Worker.max_run_time)
+          # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+          # this leads to a more even distribution of jobs across the worker processes
+          find_available(worker.name, 5, max_run_time).detect do |job|
+            job.lock_exclusively!(max_run_time, worker.name)
+          end
+        end
+
         # Hook method that is called before a new worker is forked
         def before_fork
         end
@@ -71,6 +79,16 @@ module Delayed
         self.locked_at    = nil
         self.locked_by    = nil
       end
+    
+      def reschedule_at
+        payload_object.respond_to?(:reschedule_at) ? 
+          payload_object.reschedule_at(self.class.db_time_now, attempts) :
+          self.class.db_time_now + (attempts ** 4) + 5
+      end
+
+      def max_attempts
+        payload_object.max_attempts if payload_object.respond_to?(:max_attempts)
+      end
       
     private
 
@@ -99,13 +117,13 @@ module Delayed
       def attempt_to_load(klass)
          klass.constantize
       end
-
+      
     protected
 
       def set_default_run_at
         self.run_at ||= self.class.db_time_now
-      end
-    
+      end    
+
     end
   end
 end

data/lib/delayed/command.rb

@@ -44,8 +44,9 @@ module Delayed
         opts.on('-m', '--monitor', 'Start monitor process.') do
           @monitor = true
         end
-        
-
+        opts.on('--sleep-delay N', "Amount of time to sleep when no jobs are found") do |n|
+          @options[:sleep_delay] = n
+        end
       end
       @args = opts.parse!(args)
     end

data/lib/delayed/recipes.rb

@@ -6,6 +6,17 @@
 #   after "deploy:stop",    "delayed_job:stop"
 #   after "deploy:start",   "delayed_job:start"
 #   after "deploy:restart", "delayed_job:restart"
+#
+# If you want to use command line options, for example to start multiple workers,
+# define a Capistrano variable delayed_job_args:
+#
+#   set :delayed_jobs_args, "-n 2"
+#
+# If you've got delayed_job workers running on a servers, you can also specify
+# which servers have delayed_job running and should be restarted after deploy.
+#
+#   set :delayed_job_server_role, :worker
+#
 
 Capistrano::Configuration.instance.load do
   namespace :delayed_job do
@@ -17,18 +28,22 @@ Capistrano::Configuration.instance.load do
       fetch(:delayed_job_args, "")
     end
     
+    def roles
+      fetch(:delayed_job_server_role, :app)
+    end
+    
     desc "Stop the delayed_job process"
-    task :stop, :roles => :app do
+    task :stop, :roles => lambda { roles } do
       run "cd #{current_path};#{rails_env} script/delayed_job stop"
     end
 
     desc "Start the delayed_job process"
-    task :start, :roles => :app do
+    task :start, :roles => lambda { roles } do
       run "cd #{current_path};#{rails_env} script/delayed_job start #{args}"
     end
 
     desc "Restart the delayed_job process"
-    task :restart, :roles => :app do
+    task :restart, :roles => lambda { roles } do
       run "cd #{current_path};#{rails_env} script/delayed_job restart #{args}"
     end
   end

data/lib/delayed/worker.rb

@@ -49,6 +49,7 @@ module Delayed
       @quiet = options[:quiet]
       self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
       self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+      self.class.sleep_delay = options[:sleep_delay] if options.has_key?(:sleep_delay)
     end
 
     # Every worker has a unique name which by default is the pid of the process. There are some
@@ -84,7 +85,7 @@ module Delayed
         break if $exit
 
         if count.zero?
-          sleep(@@sleep_delay)
+          sleep(self.class.sleep_delay)
         else
           say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
         end
@@ -131,9 +132,8 @@ module Delayed
     # Reschedule the job in the future (when a job fails).
     # Uses an exponential scale depending on the number of failed attempts.
     def reschedule(job, time = nil)
-      if (job.attempts += 1) < self.class.max_attempts
-        time ||= Job.db_time_now + (job.attempts ** 4) + 5
-        job.run_at = time
+      if (job.attempts += 1) < max_attempts(job)
+        job.run_at = time || job.reschedule_at
         job.unlock
         job.save!
       else
@@ -154,6 +154,10 @@ module Delayed
       logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
     end
 
+    def max_attempts(job)
+      job.max_attempts || self.class.max_attempts
+    end
+    
   protected
     
     def handle_failed_job(job, error)
@@ -165,19 +169,7 @@ module Delayed
     # Run the next job we can get an exclusive lock on.
     # If no jobs are left we return nil
     def reserve_and_run_one_job
-
-      # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
-      # this leads to a more even distribution of jobs across the worker processes
-      job = Delayed::Job.find_available(name, 5, self.class.max_run_time).detect do |job|
-        if job.lock_exclusively!(self.class.max_run_time, name)
-          say "acquired lock on #{job.name}"
-          true
-        else
-          say "failed to acquire exclusive lock for #{job.name}", Logger::WARN
-          false
-        end
-      end
-
+      job = Delayed::Job.reserve(self)
       run(job) if job
     end
   end

data/spec/backend/shared_backend_spec.rb

@@ -173,6 +173,46 @@ shared_examples_for 'a backend' do
     end
   end
 
+  describe "reserve" do
+    before do
+      Delayed::Worker.max_run_time = 2.minutes
+      @worker = Delayed::Worker.new(:quiet => true)
+    end
+
+    it "should not reserve failed jobs" do
+      create_job :attempts => 50, :failed_at => described_class.db_time_now
+      described_class.reserve(@worker).should be_nil
+    end
+
+    it "should not reserve jobs scheduled for the future" do
+      create_job :run_at => (described_class.db_time_now + 1.minute)
+      described_class.reserve(@worker).should be_nil
+    end
+
+    it "should lock the job so other workers can't reserve it" do
+      job = create_job
+      described_class.reserve(@worker).should == job
+      new_worker = Delayed::Worker.new(:quiet => true)
+      new_worker.name = 'worker2'
+      described_class.reserve(new_worker).should be_nil
+    end
+
+    it "should reserve open jobs" do
+      job = create_job
+      described_class.reserve(@worker).should == job
+    end
+
+    it "should reserve expired jobs" do
+      job = create_job(:locked_by => @worker.name, :locked_at => described_class.db_time_now - 3.minutes)
+      described_class.reserve(@worker).should == job
+    end
+
+    it "should reserve own jobs" do
+      job = create_job(:locked_by => @worker.name, :locked_at => (described_class.db_time_now - 1.minutes))
+      described_class.reserve(@worker).should == job
+    end
+  end
+  
   context "#name" do
     it "should be the class name of the job that was enqueued" do
       @backend.create(:payload_object => ErrorJob.new ).name.should == 'ErrorJob'
@@ -259,4 +299,19 @@ shared_examples_for 'a backend' do
       @job.id.should_not be_nil
     end
   end
+  
+  context "max_attempts" do
+    before(:each) do
+      @job = described_class.enqueue SimpleJob.new
+    end
+    
+    it 'should not be defined' do
+      @job.max_attempts.should be_nil
+    end
+    
+    it 'should use the max_retries value on the payload when defined' do
+      @job.payload_object.stub!(:max_attempts).and_return(99)
+      @job.max_attempts.should == 99
+    end 
+  end
 end

data/spec/sample_jobs.rb

@@ -15,6 +15,7 @@ end
 class OnPermanentFailureJob < SimpleJob
   def on_permanent_failure
   end
+  def max_attempts; 1; end
 end
 
 module M

data/spec/worker_spec.rb

@@ -131,6 +131,24 @@ describe Delayed::Worker do
           @job.locked_at.should be_nil
           @job.locked_by.should be_nil
         end
+        
+        context "when the job's payload implements #reschedule_at" do
+          before(:each) do
+            @reschedule_at = Time.current + 7.hours
+            @job.payload_object.stub!(:reschedule_at).and_return(@reschedule_at)
+          end
+
+          it 'should invoke the strategy to re-schedule' do
+            @job.payload_object.should_receive(:reschedule_at) do |time, attempts|
+              (Delayed::Job.db_time_now - time).should < 2
+              attempts.should == 1
+
+              Delayed::Job.db_time_now + 5
+            end
+
+            @worker.run(@job)
+          end
+        end
       end
   
       context "reschedule" do
@@ -147,7 +165,7 @@ describe Delayed::Worker do
 
             it "should run that hook" do
               @job.payload_object.should_receive :on_permanent_failure
-              Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
+              @worker.reschedule(@job)
             end
           end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: delayed_job
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 5
   prerelease: false
   segments: 
   - 2
   - 0
-  - 4
-  version: 2.0.4
+  - 5
+  version: 2.0.5
 platform: ruby
 authors: 
 - Brandon Keepers
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-14 00:00:00 -06:00
+date: 2010-12-01 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency