delayed_job 2.1.4 → 3.0.0.pre

data/lib/delayed/worker.rb

@@ -2,16 +2,22 @@ require 'timeout'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/class/attribute_accessors'
 require 'active_support/core_ext/kernel'
+require 'active_support/core_ext/enumerable'
 require 'logger'
 
 module Delayed
   class Worker
-    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger, :delay_jobs
+    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger, :delay_jobs, :queues
     self.sleep_delay = 5
     self.max_attempts = 25
     self.max_run_time = 4.hours
     self.default_priority = 0
     self.delay_jobs = true
+    self.queues = []
+
+    # Add or remove plugins in this list before the worker is instantiated
+    cattr_accessor :plugins
+    self.plugins = [Delayed::Plugins::ClearLocks]
 
     # By default failed jobs are destroyed after too many attempts. If you want to keep them around
     # (perhaps to inspect the reason for the failure), set this to false.
@@ -31,7 +37,7 @@ module Delayed
 
     def self.backend=(backend)
       if backend.is_a? Symbol
-        require "delayed/serialization/#{backend}"
+        require "delayed/serialization/#{backend}" if YAML.parser.class.name =~ /syck/i
         require "delayed/backend/#{backend}"
         backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
       end
@@ -40,7 +46,35 @@ module Delayed
     end
 
     def self.guess_backend
-      self.backend ||= :active_record if defined?(ActiveRecord)
+      warn "[DEPRECATION] guess_backend is deprecated. Please remove it from your code."
+    end
+
+    def self.before_fork
+      unless @files_to_reopen
+        @files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+          @files_to_reopen << file unless file.closed?
+        end
+      end
+
+      backend.before_fork
+    end
+
+    def self.after_fork
+      # Re-open file handles
+      @files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, "a+"
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      backend.after_fork
+    end
+
+    def self.lifecycle
+      @lifecycle ||= Delayed::Lifecycle.new
     end
 
     def initialize(options={})
@@ -48,6 +82,9 @@ module Delayed
       self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
       self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
       self.class.sleep_delay = options[:sleep_delay] if options.has_key?(:sleep_delay)
+      self.class.queues = options[:queues] if options.has_key?(:queues)
+
+      self.plugins.each { |klass| klass.new }
     end
 
     # Every worker has a unique name which by default is the pid of the process. There are some
@@ -66,33 +103,38 @@ module Delayed
     end
 
     def start
+      trap('TERM') { say 'Exiting...'; stop }
+      trap('INT')  { say 'Exiting...'; stop }
+
       say "Starting job worker"
 
-      trap('TERM') { say 'Exiting...'; $exit = true }
-      trap('INT')  { say 'Exiting...'; $exit = true }
+      self.class.lifecycle.run_callbacks(:execute, self) do
+        loop do
+          self.class.lifecycle.run_callbacks(:loop, self) do
+            result = nil
 
-      loop do
-        result = nil
+            realtime = Benchmark.realtime do
+              result = work_off
+            end
 
-        realtime = Benchmark.realtime do
-          result = work_off
-        end
+            count = result.sum
 
-        count = result.sum
+            break if @exit
 
-        break if $exit
+            if count.zero?
+              sleep(self.class.sleep_delay)
+            else
+              say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+            end
+          end
 
-        if count.zero?
-          sleep(self.class.sleep_delay)
-        else
-          say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+          break if @exit
         end
-
-        break if $exit
       end
+    end
 
-    ensure
-      Delayed::Job.clear_locks!(name)
+    def stop
+      @exit = true
     end
 
     # Do num jobs and return stats on success/failure.
@@ -126,7 +168,7 @@ module Delayed
       job.last_error = "{#{error.message}\n#{error.backtrace.join('\n')}"
       failed(job)
     rescue Exception => error
-      handle_failed_job(job, error)
+      self.class.lifecycle.run_callbacks(:error, self, job){ handle_failed_job(job, error) }
       return false  # work failed
     end
 
@@ -145,11 +187,10 @@ module Delayed
     end
 
     def failed(job)
-      job.hook(:failure)
-      if job.respond_to?(:on_permanent_failure)
-        warn "[DEPRECATION] The #on_permanent_failure hook has been renamed to #failure."
+      self.class.lifecycle.run_callbacks(:failure, self, job) do
+        job.hook(:failure)
+        self.class.destroy_failed_jobs ? job.destroy : job.fail!
       end
-      self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
     end
 
     def say(text, level = Logger::INFO)
@@ -161,7 +202,7 @@ module Delayed
     def max_attempts(job)
       job.max_attempts || self.class.max_attempts
     end
-    
+
   protected
 
     def handle_failed_job(job, error)
@@ -174,7 +215,7 @@ module Delayed
     # If no jobs are left we return nil
     def reserve_and_run_one_job
       job = Delayed::Job.reserve(self)
-      run(job) if job
+      self.class.lifecycle.run_callbacks(:perform, self, job){ result = run(job) } if job
     end
   end
 

data/lib/delayed/yaml_ext.rb

@@ -2,40 +2,9 @@
 # Classes, Modules and Structs
 
 require 'yaml'
-YAML::ENGINE.yamler = "syck" if defined?(YAML::ENGINE)
-
-class Module
-  yaml_as "tag:ruby.yaml.org,2002:module"
-
-  def self.yaml_new(klass, tag, val)
-    val.constantize
-  end
-
-  def to_yaml( opts = {} )
-    YAML::quick_emit( nil, opts ) { |out|
-      out.scalar(taguri, self.name, :plain)
-    }
-  end
-  
-  def yaml_tag_read_class(name)
-    # Constantize the object so that ActiveSupport can attempt
-    # its auto loading magic. Will raise LoadError if not successful.
-    name.constantize
-    name
-  end
-  
-end
-
-class Class
-  yaml_as "tag:ruby.yaml.org,2002:class"
-  remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
-end
-
-class Struct
-  def self.yaml_tag_read_class(name)
-    # Constantize the object so that ActiveSupport can attempt
-    # its auto loading magic. Will raise LoadError if not successful.
-    name.constantize
-    "Struct::#{ name }"
-  end
+if YAML.parser.class.name =~ /syck/i
+  require File.expand_path('../syck_ext', __FILE__)
+  require File.expand_path('../serialization/active_record', __FILE__)
+else
+  require File.expand_path('../psych_ext', __FILE__)
 end

data/lib/delayed_job.rb

@@ -2,8 +2,17 @@ require 'active_support'
 
 require File.dirname(__FILE__) + '/delayed/message_sending'
 require File.dirname(__FILE__) + '/delayed/performable_method'
-require File.dirname(__FILE__) + '/delayed/performable_mailer' if defined?(ActionMailer)
+
+# PerformableMailer is compatible with ActionMailer 3 (and possibly 3.1)
+if defined?(ActionMailer)
+  require 'action_mailer/version'
+  require File.dirname(__FILE__) + '/delayed/performable_mailer' if 3 == ActionMailer::VERSION::MAJOR
+end
+
 require File.dirname(__FILE__) + '/delayed/yaml_ext'
+require File.dirname(__FILE__) + '/delayed/lifecycle'
+require File.dirname(__FILE__) + '/delayed/plugin'
+require File.dirname(__FILE__) + '/delayed/plugins/clear_locks'
 require File.dirname(__FILE__) + '/delayed/backend/base'
 require File.dirname(__FILE__) + '/delayed/worker'
 require File.dirname(__FILE__) + '/delayed/deserialization_error'

data/lib/generators/delayed_job/delayed_job_generator.rb

@@ -1,34 +1,11 @@
 require 'rails/generators'
-require 'rails/generators/migration'
 
 class DelayedJobGenerator < Rails::Generators::Base
-
-  include Rails::Generators::Migration
   
-  def self.source_root
-     @source_root ||= File.join(File.dirname(__FILE__), 'templates')
-  end
-
-  # Implement the required interface for Rails::Generators::Migration.
-  #
-  def self.next_migration_number(dirname) #:nodoc:
-    next_migration_number = current_migration_number(dirname) + 1
-    if ActiveRecord::Base.timestamped_migrations
-      [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
-    else
-      "%.3d" % next_migration_number
-    end
-  end
+  self.source_paths << File.join(File.dirname(__FILE__), 'templates')
   
   def create_script_file
     template 'script', 'script/delayed_job'
     chmod 'script/delayed_job', 0755
   end
-  
-  def create_migration_file
-    if defined?(ActiveRecord)
-      migration_template 'migration.rb', 'db/migrate/create_delayed_jobs.rb'
-    end
-  end
-
 end

data/spec/autoloaded/instance_clazz.rb

@@ -0,0 +1,6 @@
+module Autoloaded
+  class InstanceClazz
+    def perform
+    end
+  end
+end

data/spec/autoloaded/instance_struct.rb

@@ -0,0 +1,6 @@
+module Autoloaded
+  class InstanceStruct < ::Struct.new(nil)
+    def perform
+    end
+  end
+end

data/spec/delayed/backend/test.rb

@@ -0,0 +1,112 @@
+require 'ostruct'
+
+# An in-memory backend suitable only for testing. Tries to behave as if it were an ORM.
+module Delayed
+  module Backend
+    module Test
+      class Job
+        attr_accessor :id
+        attr_accessor :priority
+        attr_accessor :attempts
+        attr_accessor :handler
+        attr_accessor :last_error
+        attr_accessor :run_at
+        attr_accessor :locked_at
+        attr_accessor :locked_by
+        attr_accessor :failed_at
+        attr_accessor :queue
+        
+        include Delayed::Backend::Base
+
+        cattr_accessor :id
+        self.id = 0
+        
+        def initialize(hash = {})
+          self.attempts = 0
+          self.priority = 0
+          self.id = (self.class.id += 1)
+          hash.each{|k,v| send(:"#{k}=", v)}
+        end
+        
+        @jobs = []
+        def self.all
+          @jobs
+        end
+        
+        def self.count 
+          all.size
+        end
+        
+        def self.delete_all
+          all.clear
+        end
+        
+        def self.create(attrs = {})
+          new(attrs).tap do |o|
+            o.save
+          end
+        end
+        
+        def self.create!(*args); create(*args); end
+        
+        def self.clear_locks!(worker_name)
+          all.select{|j| j.locked_by == worker_name}.each {|j| j.locked_by = nil; j.locked_at = nil}
+        end
+
+        # Find a few candidate jobs to run (in case some immediately get locked by others).
+        def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
+          jobs = all.select do |j| 
+            j.run_at <= db_time_now && 
+            (j.locked_at.nil? || j.locked_at < db_time_now - max_run_time || j.locked_by == worker_name) &&
+            j.failed_at.nil?
+          end
+                    
+          jobs = jobs.select{|j| Worker.queues.include?(j.queue)}   if Worker.queues.any?
+          jobs = jobs.select{|j| j.priority >= Worker.min_priority} if Worker.min_priority
+          jobs = jobs.select{|j| j.priority <= Worker.max_priority} if Worker.max_priority          
+          jobs.sort_by{|j| [j.priority, j.run_at]}[0..limit-1]
+        end
+
+        # Lock this job for this worker.
+        # Returns true if we have the lock, false otherwise.
+        def lock_exclusively!(max_run_time, worker)
+          now = self.class.db_time_now
+          if locked_by != worker
+            # We don't own this job so we will update the locked_by name and the locked_at
+            self.locked_at = now
+            self.locked_by = worker
+          end
+
+          return true
+        end
+
+        def self.db_time_now
+          Time.current
+        end
+        
+        def update_attributes(attrs = {})
+          attrs.each{|k,v| send(:"#{k}=", v)}
+          save
+        end
+        
+        def destroy
+          self.class.all.delete(self)
+        end
+        
+        def save
+          self.run_at ||= Time.current
+          
+          self.class.all << self unless self.class.all.include?(self)
+          true
+        end
+        
+        def save!; save; end
+               
+        def reload
+          reset
+          self
+        end
+      end
+    end
+  end
+end

data/spec/lifecycle_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+describe Delayed::Lifecycle do
+  let(:lifecycle) { Delayed::Lifecycle.new }
+  let(:callback) { lambda {|*args|} }
+  let(:arguments) { [1] }
+  let(:behavior) { mock(Object, :before! => nil, :after! => nil, :inside! => nil) }
+  let(:wrapped_block) { Proc.new { behavior.inside! } }
+
+  describe "before callbacks" do
+    before(:each) do
+      lifecycle.before(:execute, &callback)
+    end
+
+    it 'should execute before wrapped block' do
+      callback.should_receive(:call).with(*arguments).ordered
+      behavior.should_receive(:inside!).ordered
+      lifecycle.run_callbacks :execute, *arguments, &wrapped_block
+    end
+  end
+
+  describe "after callbacks" do
+    before(:each) do
+      lifecycle.after(:execute, &callback)
+    end
+
+    it 'should execute after wrapped block' do
+      behavior.should_receive(:inside!).ordered
+      callback.should_receive(:call).with(*arguments).ordered
+      lifecycle.run_callbacks :execute, *arguments, &wrapped_block
+    end
+  end
+
+  describe "around callbacks" do
+    before(:each) do
+      lifecycle.around(:execute) do |*args, &block|
+        behavior.before!
+        block.call(*args)
+        behavior.after!
+      end
+    end
+
+    it 'should before and after wrapped block' do
+      behavior.should_receive(:before!).ordered
+      behavior.should_receive(:inside!).ordered
+      behavior.should_receive(:after!).ordered
+      lifecycle.run_callbacks :execute, *arguments, &wrapped_block
+    end
+
+    it "should execute multiple callbacks in order" do
+      behavior.should_receive(:one).ordered
+      behavior.should_receive(:two).ordered
+      behavior.should_receive(:three).ordered
+
+      lifecycle.around(:execute) { |*args, &block| behavior.one; block.call(*args) }
+      lifecycle.around(:execute) { |*args, &block| behavior.two; block.call(*args) }
+      lifecycle.around(:execute) { |*args, &block| behavior.three; block.call(*args) }
+
+      lifecycle.run_callbacks(:execute, *arguments, &wrapped_block)
+    end
+  end
+
+  it "should raise if callback is executed with wrong number of parameters" do
+    lifecycle.before(:execute, &callback)
+    expect { lifecycle.run_callbacks(:execute, 1,2,3) {} }.to raise_error(ArgumentError, /1 parameter/)
+  end
+
+  # # This is a spectacularly crappy way to test callbacks. What's a better way?
+  # describe 'arguments callbacks' do
+  #   subject do
+  #     class Testarguments < Delayed::arguments
+  #       def before_execute; end
+  #       def before_loop; end
+  #       def before_perform; end
+  #
+  #       set_callback :execute, :before, :before_execute
+  #       set_callback :loop, :before, :before_loop
+  #       set_callback :perform, :before, :before_perform
+  #     end
+  #
+  #     Testarguments.new.tap { |w| w.stop }
+  #   end
+  #
+  #   it "should trigger for execute event" do
+  #     subject.should_receive(:before_execute).with()
+  #     subject.start
+  #   end
+  #
+  #   it "should trigger for loop event" do
+  #     subject.should_receive(:before_loop).with()
+  #     subject.start
+  #   end
+  #
+  #   it "should trigger for perform event" do
+  #     "foo".delay.length
+  #     subject.should_receive(:before_perform).with()
+  #     subject.start
+  #   end
+  # end
+  #
+  # describe 'job callbacks' do
+  #   it "should trigger for enqueue event" do
+  #     pending 'figure out how to test this'
+  #   end
+  # end
+
+end

data/spec/message_sending_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Delayed::MessageSending do
   describe "handle_asynchronously" do
-    class Story < ActiveRecord::Base
+    class Story
       def tell!(arg)
       end
       handle_asynchronously :tell!
@@ -14,7 +14,7 @@ describe Delayed::MessageSending do
     end
 
     it "should create a PerformableMethod" do
-      story = Story.create!
+      story = Story.new
       lambda {
         job = story.tell!(1)
         job.payload_object.class.should   == Delayed::PerformableMethod

data/spec/performable_method_spec.rb

@@ -47,18 +47,43 @@ describe Delayed::PerformableMethod do
         story.delay.tell.invoke_job
       end
     end
-
+    
+    %w(before after success).each do |hook|
+      it "should delegate #{hook} hook to object when delay_jobs = false" do
+        Delayed::Worker.delay_jobs = false
+        story = Story.new
+        story.should_receive(hook).with(an_instance_of(Delayed::Job))
+        story.delay.tell
+      end
+    end
+    
     it "should delegate error hook to object" do
       story = Story.new
       story.should_receive(:error).with(an_instance_of(Delayed::Job), an_instance_of(RuntimeError))
       story.should_receive(:tell).and_raise(RuntimeError)
       lambda { story.delay.tell.invoke_job }.should raise_error
     end
+    
+    it "should delegate error hook to object when delay_jobs = false" do
+      Delayed::Worker.delay_jobs = false
+      story = Story.new
+      story.should_receive(:error).with(an_instance_of(Delayed::Job), an_instance_of(RuntimeError))
+      story.should_receive(:tell).and_raise(RuntimeError)
+      lambda { story.delay.tell }.should raise_error
+    end
 
     it "should delegate failure hook to object" do
       method = Delayed::PerformableMethod.new("object", :size, [])
       method.object.should_receive(:failure)
       method.failure
     end
+    
+    it "should delegate failure hook to object when delay_jobs = false" do
+      Delayed::Worker.delay_jobs = false
+      method = Delayed::PerformableMethod.new("object", :size, [])
+      method.object.should_receive(:failure)
+      method.failure
+    end
+    
   end
 end

data/spec/sample_jobs.rb

@@ -67,3 +67,9 @@ class CallbackJob
     self.class.messages << 'failure'
   end
 end
+
+class EnqueueJobMod < SimpleJob
+  def enqueue(job)
+    job.run_at = 20.minutes.from_now
+  end
+end

data/spec/spec_helper.rb

@@ -6,8 +6,9 @@ require 'rspec'
 require 'logger'
 
 require 'rails'
-require 'active_record'
 require 'action_mailer'
+require 'active_support/dependencies'
+require 'active_record'
 
 require 'delayed_job'
 require 'delayed/backend/shared_spec'
@@ -15,44 +16,32 @@ require 'delayed/backend/shared_spec'
 Delayed::Worker.logger = Logger.new('/tmp/dj.log')
 ENV['RAILS_ENV'] = 'test'
 
-config = YAML.load(File.read('spec/database.yml'))
-ActiveRecord::Base.configurations = {'test' => config['mysql']}
-ActiveRecord::Base.establish_connection
+Delayed::Worker.backend = :test
+
+# Add this directory so the ActiveSupport autoloading works
+ActiveSupport::Dependencies.autoload_paths << File.dirname(__FILE__)
+
+# Add this to simulate Railtie initializer being executed
+ActionMailer::Base.send(:extend, Delayed::DelayMail)
+
+
+# Used to test interactions between DJ and an ORM
+ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => ':memory:'
 ActiveRecord::Base.logger = Delayed::Worker.logger
 ActiveRecord::Migration.verbose = false
 
 ActiveRecord::Schema.define do
-  create_table :delayed_jobs, :force => true do |table|
-    table.integer  :priority, :default => 0
-    table.integer  :attempts, :default => 0
-    table.text     :handler
-    table.text     :last_error
-    table.datetime :run_at
-    table.datetime :locked_at
-    table.datetime :failed_at
-    table.string   :locked_by
-    table.timestamps
-  end
-
-  add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
-
-  create_table :stories, :force => true do |table|
+  create_table :stories, :primary_key => :story_id, :force => true do |table|
     table.string :text
+    table.boolean :scoped, :default => true
   end
 end
 
-# Purely useful for test cases...
 class Story < ActiveRecord::Base
+  set_primary_key :story_id
   def tell; text; end
   def whatever(n, _); tell*n; end
+  default_scope where(:scoped => true)
 
   handle_asynchronously :whatever
 end
-
-Delayed::Worker.backend = :active_record
-
-# Add this directory so the ActiveSupport autoloading works
-ActiveSupport::Dependencies.autoload_paths << File.dirname(__FILE__)
-
-# Add this to simulate Railtie initializer being executed
-ActionMailer::Base.send(:extend, Delayed::DelayMail)

data/spec/test_backend_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Delayed::Backend::Test::Job do
+  it_should_behave_like 'a delayed_job backend'
+  
+  describe "#reload" do
+    it 'should cause the payload object to be reloaded' do
+      job = "foo".delay.length
+      o = job.payload_object
+      o.object_id.should_not == job.reload.payload_object.object_id
+    end
+  end
+end