delayed_job 2.0.1 → 2.0.2

data/.gitignore

@@ -1 +1,2 @@
 *.gem
+*.swp

data/README.textile

@@ -123,6 +123,20 @@ end
 Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email))
 </pre>
 
+You can also add an optional on_permanent_failure method which will run if the job has failed too many times to be retried:
+
+<pre>
+class ParanoidNewsletterJob < NewsletterJob
+  def perform
+    emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
+  end    
+
+  def on_permanent_failure
+    page_sysadmin_in_the_middle_of_the_night
+  end
+end  
+</pre>
+
 h2. Gory Details
 
 The library evolves around a delayed_jobs table which looks as follows: 

data/Rakefile

@@ -27,6 +27,7 @@ Jeweler::Tasks.new do |s|
   s.add_development_dependency "dm-core"
   s.add_development_dependency "dm-observer"
   s.add_development_dependency "dm-aggregates"
+  s.add_development_dependency "dm-validations"
   s.add_development_dependency "do_sqlite3"
   s.add_development_dependency "database_cleaner"
 end

data/VERSION

@@ -1 +1 @@
-2.0.1
+2.0.2

data/benchmarks.rb

@@ -6,8 +6,20 @@ require 'benchmark'
 
 Delayed::Worker.logger = Logger.new('/dev/null')
 
+BACKENDS = []
+Dir.glob("#{File.dirname(__FILE__)}/spec/setup/*.rb") do |backend|
+  begin
+    backend = File.basename(backend, '.rb')
+    require "spec/setup/#{backend}"
+    BACKENDS << backend.to_sym
+  rescue LoadError
+    puts "Unable to load #{backend} backend! #{$!}"
+  end
+end
+
+
 Benchmark.bm(10) do |x|
-  [:active_record, :mongo_mapper, :data_mapper].each do |backend|
+  BACKENDS.each do |backend|
     require "spec/setup/#{backend}"
     Delayed::Worker.backend = backend
   

data/delayed_job.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{delayed_job}
-  s.version = "2.0.1"
+  s.version = "2.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brandon Keepers", "Tobias L\303\274tke"]
-  s.date = %q{2010-04-03}
+  s.date = %q{2010-04-08}
   s.description = %q{Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks.
 
 This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job).}
@@ -29,6 +29,7 @@ This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job)
      "generators/delayed_job/delayed_job_generator.rb",
      "generators/delayed_job/templates/migration.rb",
      "generators/delayed_job/templates/script",
+     "init.rb",
      "lib/delayed/backend/active_record.rb",
      "lib/delayed/backend/base.rb",
      "lib/delayed/backend/data_mapper.rb",
@@ -36,11 +37,11 @@ This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job)
      "lib/delayed/command.rb",
      "lib/delayed/message_sending.rb",
      "lib/delayed/performable_method.rb",
+     "lib/delayed/railtie.rb",
      "lib/delayed/recipes.rb",
      "lib/delayed/tasks.rb",
      "lib/delayed/worker.rb",
      "lib/delayed_job.rb",
-     "rails/init.rb",
      "recipes/delayed_job.rb",
      "spec/backend/active_record_job_spec.rb",
      "spec/backend/data_mapper_job_spec.rb",
@@ -81,6 +82,7 @@ This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job)
       s.add_development_dependency(%q<dm-core>, [">= 0"])
       s.add_development_dependency(%q<dm-observer>, [">= 0"])
       s.add_development_dependency(%q<dm-aggregates>, [">= 0"])
+      s.add_development_dependency(%q<dm-validations>, [">= 0"])
       s.add_development_dependency(%q<do_sqlite3>, [">= 0"])
       s.add_development_dependency(%q<database_cleaner>, [">= 0"])
     else
@@ -91,6 +93,7 @@ This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job)
       s.add_dependency(%q<dm-core>, [">= 0"])
       s.add_dependency(%q<dm-observer>, [">= 0"])
       s.add_dependency(%q<dm-aggregates>, [">= 0"])
+      s.add_dependency(%q<dm-validations>, [">= 0"])
       s.add_dependency(%q<do_sqlite3>, [">= 0"])
       s.add_dependency(%q<database_cleaner>, [">= 0"])
     end
@@ -102,6 +105,7 @@ This gem is collectiveidea's fork (http://github.com/collectiveidea/delayed_job)
     s.add_dependency(%q<dm-core>, [">= 0"])
     s.add_dependency(%q<dm-observer>, [">= 0"])
     s.add_dependency(%q<dm-aggregates>, [">= 0"])
+    s.add_dependency(%q<dm-validations>, [">= 0"])
     s.add_dependency(%q<do_sqlite3>, [">= 0"])
     s.add_dependency(%q<database_cleaner>, [">= 0"])
   end

data/init.rb

@@ -0,0 +1,5 @@
+require 'delayed_job'
+
+config.after_initialize do
+  Delayed::Worker.guess_backend
+end

data/lib/delayed/backend/data_mapper.rb

@@ -28,14 +28,14 @@ module Delayed
         storage_names[:default] = 'delayed_jobs'
         
         property :id,          Serial
-        property :priority,    Integer, :default => 0
+        property :priority,    Integer, :default => 0, :index => :run_at_priority
         property :attempts,    Integer, :default => 0
-        property :handler,     String
-        property :run_at,      Time
-        property :locked_at,   Time
+        property :handler,     Text, :lazy => false
+        property :run_at,      Time, :index => :run_at_priority
+        property :locked_at,   Time, :index => true
         property :locked_by,   String
         property :failed_at,   Time
-        property :last_error,  String
+        property :last_error,  Text
                 
         def self.db_time_now
           Time.now
@@ -69,6 +69,7 @@ module Delayed
         # Lock this job for this worker.
         # Returns true if we have the lock, false otherwise.
         def lock_exclusively!(max_run_time, worker = worker_name)
+
           now = self.class.db_time_now
           overtime = now - max_run_time
           

data/lib/delayed/backend/mongo_mapper.rb

@@ -43,11 +43,11 @@ module Delayed
         end
         
         def self.after_fork
-          ::MongoMapper.connection.connect_to_master
+          ::MongoMapper.connect(RAILS_ENV)
         end
         
         def self.db_time_now
-          ::MongoMapper.time_class.now.utc
+          Time.now.utc
         end
         
         def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
@@ -62,8 +62,8 @@ module Delayed
 
           where = "this.locked_at == null || this.locked_at < #{make_date(right_now - max_run_time)}"
           
-          (conditions[:priority] ||= {})['$gte'] = Worker.min_priority if Worker.min_priority
-          (conditions[:priority] ||= {})['$lte'] = Worker.max_priority if Worker.max_priority
+          (conditions[:priority] ||= {})['$gte'] = Worker.min_priority.to_i if Worker.min_priority
+          (conditions[:priority] ||= {})['$lte'] = Worker.max_priority.to_i if Worker.max_priority
 
           results = all(conditions.merge(:locked_by => worker_name))
           results += all(conditions.merge('$where' => where)) if results.size < limit

data/lib/delayed/command.rb

@@ -42,9 +42,12 @@ module Delayed
         @files_to_reopen << file unless file.closed?
       end
       
+      dir = "#{RAILS_ROOT}/tmp/pids"
+      Dir.mkdir(dir) unless File.exists?(dir)
+      
       worker_count.times do |worker_index|
         process_name = worker_count == 1 ? "delayed_job" : "delayed_job.#{worker_index}"
-        Daemons.run_proc(process_name, :dir => "#{RAILS_ROOT}/tmp/pids", :dir_mode => :normal, :ARGV => @args) do |*args|
+        Daemons.run_proc(process_name, :dir => dir, :dir_mode => :normal, :ARGV => @args) do |*args|
           run process_name
         end
       end

data/lib/delayed/railtie.rb

@@ -0,0 +1,10 @@
+require 'delayed_job'
+require 'rails'
+
+module Delayed
+  class Railtie < Rails::Railtie
+    initializer :after_initialize do
+      Delayed::Worker.guess_backend
+    end
+  end
+end

data/lib/delayed/worker.rb

@@ -32,6 +32,17 @@ module Delayed
       @@backend = backend
       silence_warnings { ::Delayed.const_set(:Job, backend) }
     end
+    
+    def self.guess_backend
+      self.backend ||= if defined?(ActiveRecord)
+        :active_record
+      elsif defined?(MongoMapper)
+        :mongo_mapper
+      else
+        logger.warn "Could not decide on a backend, defaulting to active_record"
+        :active_record
+      end
+    end
 
     def initialize(options={})
       @quiet = options[:quiet]
@@ -127,6 +138,12 @@ module Delayed
         job.save!
       else
         say "* [JOB] PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+
+        if job.payload_object.respond_to? :on_permanent_failure
+          say "* [JOB] Running on_permanent_failure hook"
+          job.payload_object.on_permanent_failure
+        end
+
         self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
       end
     end

data/lib/delayed_job.rb

@@ -4,6 +4,7 @@ require File.dirname(__FILE__) + '/delayed/message_sending'
 require File.dirname(__FILE__) + '/delayed/performable_method'
 require File.dirname(__FILE__) + '/delayed/backend/base'
 require File.dirname(__FILE__) + '/delayed/worker'
+require File.dirname(__FILE__) + '/delayed/railtie' if defined?(::Rails::Railtie)
 
 Object.send(:include, Delayed::MessageSending)   
 Module.send(:include, Delayed::MessageSending::ClassMethods)

data/spec/backend/shared_backend_spec.rb

@@ -243,4 +243,23 @@ shared_examples_for 'a backend' do
     end
   end
   
+  context "large handler" do
+    @@text = %{Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus eu vehicula augue. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Quisque odio lectus, volutpat sed dictum rutrum, interdum aliquam neque. Vivamus quis velit nisi, quis dictum purus. Duis magna nisi, faucibus nec molestie vitae, dictum eget odio. Nunc nulla mauris, vestibulum at dapibus nec, dapibus et lectus. Nullam sapien lacus, consectetur eget mattis in, rhoncus sed ipsum. Nullam nec nibh nisl. Integer ut erat in arcu feugiat semper. Nulla gravida sapien quam. Vestibulum pharetra elementum posuere. Fusce mattis justo auctor nibh facilisis vitae consectetur nibh vehicula.
+
+    Ut at pharetra justo. Donec dictum ornare tortor in feugiat. Sed ac purus sem. Aenean dignissim, erat vel bibendum mollis, elit neque mollis mauris, vitae pretium diam enim non leo. Aliquam aliquet, odio id iaculis varius, metus nibh fermentum sapien, a euismod turpis lectus sit amet turpis. Morbi sapien est, scelerisque in placerat in, varius nec mauris. Aliquam erat volutpat. Quisque suscipit tincidunt libero, sed tincidunt libero iaculis et. Vivamus sed faucibus elit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus dignissim sem sed tortor semper et lacinia leo viverra. Nulla nec quam at arcu ullamcorper imperdiet vitae in ligula. Quisque placerat vulputate orci sit amet tempor. Duis sed quam nulla. Cras quis mi nibh, at euismod velit. Etiam nec nunc libero, sed condimentum diam.
+
+    Duis nec mauris in est suscipit viverra a in nibh. Suspendisse nec nulla tortor. Etiam et nulla tellus. Nam feugiat adipiscing commodo. Curabitur scelerisque varius lacus non hendrerit. Vivamus nec enim non turpis auctor tempus sit amet in nisi. Sed ligula nulla, condimentum sed tempor vel, imperdiet id mauris. Quisque mollis ante eu magna tempus porttitor. Integer est libero, consectetur sed tristique a, scelerisque id risus. Donec lacinia justo eget diam fringilla vitae egestas dolor feugiat. Vivamus massa ante, mattis et hendrerit nec, dictum vitae nulla. Pellentesque at nisl et odio suscipit ullamcorper cursus quis enim. Ut nec tellus molestie erat dignissim mollis. Curabitur quis ipsum sapien, sed tincidunt massa. Vestibulum volutpat pretium fringilla.
+
+    Integer at lorem sit amet nibh suscipit euismod et ut ante. Maecenas feugiat hendrerit dolor, eget egestas velit consequat eget. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Suspendisse ut nunc odio. Vivamus semper, sem vitae sollicitudin auctor, leo mi vulputate augue, eget venenatis libero nunc ut dolor. Phasellus vulputate, metus et dapibus tempus, tellus arcu ullamcorper leo, porttitor dictum lectus turpis blandit sapien. Pellentesque et accumsan justo. Maecenas elit nisi, tincidunt eget consequat a, laoreet et magna. Pellentesque venenatis felis ut massa ultrices bibendum. Duis vulputate tempor leo at bibendum. Curabitur aliquet, turpis sit amet porta porttitor, nibh mi vehicula dolor, suscipit aliquet mi augue quis magna. Praesent tellus turpis, malesuada at ultricies id, feugiat a urna. Curabitur sed mi magna.
+
+    Quisque adipiscing dignissim mollis. Aenean blandit, diam porttitor bibendum bibendum, leo neque tempus risus, in rutrum dolor elit a lorem. Aenean sollicitudin scelerisque ullamcorper. Nunc tristique ultricies nunc et imperdiet. Duis vitae egestas mauris. Suspendisse odio nisi, accumsan vel volutpat nec, aliquam vitae odio. Praesent elementum fermentum suscipit. Quisque quis tellus eu tellus bibendum luctus a quis nunc. Praesent dictum velit sed lacus dapibus ut ultricies mauris facilisis. Vivamus bibendum, ipsum sit amet facilisis consequat, leo lectus aliquam augue, eu consectetur magna nunc gravida sapien. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Duis tempor nisl ac odio molestie ut tincidunt purus varius. Nunc quis lorem nibh, vestibulum cursus lorem. Nunc sit amet est ut magna suscipit tempor vitae a augue.}
+
+    before do
+      @job = @backend.enqueue Delayed::PerformableMethod.new(@@text, :length, {})
+    end
+    
+    it "should have an id" do
+      @job.id.should_not be_nil
+    end
+  end
 end

data/spec/sample_jobs.rb

@@ -12,10 +12,14 @@ class LongRunningJob
   def perform; sleep 250; end
 end
 
+class OnPermanentFailureJob < SimpleJob
+  def on_permanent_failure
+  end
+end
+
 module M
   class ModuleJob
     cattr_accessor :runs; self.runs = 0
     def perform; @@runs += 1; end    
   end
-  
-end
+end

data/spec/setup/data_mapper.rb

@@ -1,4 +1,6 @@
 require 'dm-core'
+require 'dm-validations'
+
 require 'delayed/backend/data_mapper'
 
 DataMapper.logger = Delayed::Worker.logger

data/spec/worker_spec.rb

@@ -134,12 +134,52 @@ describe Delayed::Worker do
         before do
           @job = Delayed::Job.create :payload_object => SimpleJob.new
         end
-    
+   
+        share_examples_for "any failure more than Worker.max_attempts times" do
+          context "when the job's payload has an #on_permanent_failure hook" do
+            before do
+              @job = Delayed::Job.create :payload_object => OnPermanentFailureJob.new
+              @job.payload_object.should respond_to :on_permanent_failure
+            end
+
+            it "should run that hook" do
+              @job.payload_object.should_receive :on_permanent_failure
+              Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
+            end
+          end
+
+          context "when the job's payload has no #on_permanent_failure hook" do
+            # It's a little tricky to test this in a straightforward way, 
+            # because putting a should_not_receive expectation on 
+            # @job.payload_object.on_permanent_failure makes that object
+            # incorrectly return true to 
+            # payload_object.respond_to? :on_permanent_failure, which is what
+            # reschedule uses to decide whether to call on_permanent_failure.  
+            # So instead, we just make sure that the payload_object as it 
+            # already stands doesn't respond_to? on_permanent_failure, then
+            # shove it through the iterated reschedule loop and make sure we
+            # don't get a NoMethodError (caused by calling that nonexistent
+            # on_permanent_failure method).
+            
+            before do
+              @job.payload_object.should_not respond_to(:on_permanent_failure)
+            end
+
+            it "should not try to run that hook" do
+              lambda do
+                Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
+              end.should_not raise_exception(NoMethodError)
+            end
+          end
+        end
+
         context "and we want to destroy jobs" do
           before do
             Delayed::Worker.destroy_failed_jobs = true
           end
-      
+
+          it_should_behave_like "any failure more than Worker.max_attempts times"
+
           it "should be destroyed if it failed more than Worker.max_attempts times" do
             @job.should_receive(:destroy)
             Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
@@ -156,6 +196,8 @@ describe Delayed::Worker do
             Delayed::Worker.destroy_failed_jobs = false
           end
       
+          it_should_behave_like "any failure more than Worker.max_attempts times"
+
           it "should be failed if it failed more than Worker.max_attempts times" do
             @job.reload.failed_at.should == nil
             Delayed::Worker.max_attempts.times { @worker.reschedule(@job) }
@@ -166,7 +208,6 @@ describe Delayed::Worker do
             (Delayed::Worker.max_attempts - 1).times { @worker.reschedule(@job) }
             @job.reload.failed_at.should == nil
           end
-      
         end
       end
     end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 2
   - 0
-  - 1
-  version: 2.0.1
+  - 2
+  version: 2.0.2
 platform: ruby
 authors: 
 - Brandon Keepers
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-03 00:00:00 -04:00
+date: 2010-04-08 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -103,7 +103,7 @@ dependencies:
   type: :development
   version_requirements: *id007
 - !ruby/object:Gem::Dependency 
-  name: do_sqlite3
+  name: dm-validations
   prerelease: false
   requirement: &id008 !ruby/object:Gem::Requirement 
     requirements: 
@@ -115,7 +115,7 @@ dependencies:
   type: :development
   version_requirements: *id008
 - !ruby/object:Gem::Dependency 
-  name: database_cleaner
+  name: do_sqlite3
   prerelease: false
   requirement: &id009 !ruby/object:Gem::Requirement 
     requirements: 
@@ -126,6 +126,18 @@ dependencies:
         version: "0"
   type: :development
   version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: database_cleaner
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id010
 description: |-
   Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background. It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks.
   
@@ -149,6 +161,7 @@ files:
 - generators/delayed_job/delayed_job_generator.rb
 - generators/delayed_job/templates/migration.rb
 - generators/delayed_job/templates/script
+- init.rb
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
 - lib/delayed/backend/data_mapper.rb
@@ -156,11 +169,11 @@ files:
 - lib/delayed/command.rb
 - lib/delayed/message_sending.rb
 - lib/delayed/performable_method.rb
+- lib/delayed/railtie.rb
 - lib/delayed/recipes.rb
 - lib/delayed/tasks.rb
 - lib/delayed/worker.rb
 - lib/delayed_job.rb
-- rails/init.rb
 - recipes/delayed_job.rb
 - spec/backend/active_record_job_spec.rb
 - spec/backend/data_mapper_job_spec.rb

data/rails/init.rb

@@ -1,12 +0,0 @@
-require 'delayed_job'
-
-config.after_initialize do
-  Delayed::Worker.backend ||= if defined?(ActiveRecord)
-    :active_record
-  elsif defined?(MongoMapper)
-    :mongo_mapper
-  else
-    Delayed::Worker.logger.warn "Could not decide on a backend, defaulting to active_record"
-    :active_record
-  end
-end