delayed_job 2.1.4 → 3.0.0.pre

data/README.textile

@@ -1,6 +1,6 @@
 h1. Delayed::Job
 
-Delated_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background.
+Delayed_job (or DJ) encapsulates the common pattern of asynchronously executing longer tasks in the background.
 
 It is a direct extraction from Shopify where the job table is responsible for a multitude of core tasks. Amongst those tasks are:
 
@@ -12,6 +12,9 @@ It is a direct extraction from Shopify where the job table is responsible for a
 * batch imports
 * spam checks
 
+"Follow us on Twitter":https://twitter.com/delayedjob to get updates and notices about new releases.
+
+
 h2. Installation
 
 delayed_job 2.1 only supports Rails 3.0+. See the "2.0 branch":https://github.com/collectiveidea/delayed_job/tree/v2.0 for Rails 2.

data/lib/delayed/backend/base.rb

@@ -23,13 +23,18 @@ module Delayed
           unless options[:payload_object].respond_to?(:perform)
             raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
           end
-          
+
           if Delayed::Worker.delay_jobs
-            self.create(options).tap do |job|
-              job.hook(:enqueue)
+            self.new(options).tap do |job|
+              Delayed::Worker.lifecycle.run_callbacks(:enqueue, job) do
+                job.hook(:enqueue)
+                job.save
+              end
             end
           else
-            options[:payload_object].perform
+            Delayed::Job.new(:payload_object => options[:payload_object]).tap do |job|
+              job.invoke_job
+            end
           end
         end
 
@@ -83,14 +88,18 @@ module Delayed
       end
 
       def invoke_job
-        hook :before
-        payload_object.perform
-        hook :success
-      rescue Exception => e
-        hook :error, e
-        raise e
-      ensure
-        hook :after
+        Delayed::Worker.lifecycle.run_callbacks(:invoke_job, self) do
+          begin
+            hook :before
+            payload_object.perform
+            hook :success
+          rescue Exception => e
+            hook :error, e
+            raise e
+          ensure
+            hook :after
+          end
+        end
       end
 
       # Unlock this job (note: not saved to DB)
@@ -113,16 +122,25 @@ module Delayed
           payload_object.reschedule_at(self.class.db_time_now, attempts) :
           self.class.db_time_now + (attempts ** 4) + 5
       end
-      
+
       def max_attempts
         payload_object.max_attempts if payload_object.respond_to?(:max_attempts)
       end
-      
+
+      def fail!
+        update_attributes(:failed_at => self.class.db_time_now)
+      end
+
     protected
 
       def set_default_run_at
         self.run_at ||= self.class.db_time_now
       end
+
+      # Call during reload operation to clear out internal state
+      def reset
+        @payload_object = nil
+      end
     end
   end
 end

data/lib/delayed/backend/shared_spec.rb

@@ -1,5 +1,7 @@
 require File.expand_path('../../../../spec/sample_jobs', __FILE__)
 
+require 'active_support/core_ext'
+
 shared_examples_for 'a delayed_job backend' do
   let(:worker) { Delayed::Worker.new }
 
@@ -26,6 +28,13 @@ shared_examples_for 'a delayed_job backend' do
     job.run_at.should be_within(1).of(later)
   end
 
+  describe "#reload" do
+    it 'should cause the payload to be reloaded' do
+      job = described_class.enqueue :payload_object => SimpleJob.new
+      job.payload_object.object_id.should_not == job.reload.payload_object.object_id
+    end
+  end
+
   describe "enqueue" do
     context "with a hash" do
       it "should raise ArgumentError when handler doesn't respond_to :perform" do
@@ -47,6 +56,11 @@ shared_examples_for 'a delayed_job backend' do
         job = described_class.enqueue :payload_object => SimpleJob.new, :run_at => later
         job.run_at.should be_within(1).of(later)
       end
+
+      it "should be able to set queue" do
+        job = described_class.enqueue :payload_object => SimpleJob.new, :queue => 'tracking'
+        job.queue.should == 'tracking'
+      end
     end
 
     context "with multiple arguments" do
@@ -58,12 +72,6 @@ shared_examples_for 'a delayed_job backend' do
         described_class.enqueue SimpleJob.new
         described_class.count.should == 1
       end
-      
-      it "should not increase count after enqueuing items when delay_jobs is false" do
-        Delayed::Worker.delay_jobs = false
-        described_class.enqueue SimpleJob.new
-        described_class.count.should == 0
-      end
 
       it "should be able to set priority [DEPRECATED]" do
         silence_warnings do
@@ -91,6 +99,27 @@ shared_examples_for 'a delayed_job backend' do
         lambda { job.invoke_job }.should change { M::ModuleJob.runs }.from(0).to(1)
       end
     end
+
+    context "with delay_jobs = false" do
+      before(:each) do
+        Delayed::Worker.delay_jobs = false
+      end
+
+      it "should not increase count after enqueuing items" do
+        described_class.enqueue SimpleJob.new
+        described_class.count.should == 0
+      end
+
+      it 'should invoke the enqueued job' do
+        job = SimpleJob.new
+        job.should_receive(:perform)
+        described_class.enqueue job
+      end
+
+      it 'should return a job, not the result of invocation' do
+        described_class.enqueue(SimpleJob.new).should be_instance_of(described_class)
+      end
+    end
   end
 
   describe "callbacks" do
@@ -141,7 +170,7 @@ shared_examples_for 'a delayed_job backend' do
     end
 
     it "should raise a DeserializationError when the YAML.load raises argument error" do
-      job = described_class.find(create_job.id)
+      job = described_class.new :handler => "--- !ruby/struct:GoingToRaiseArgError {}"
       YAML.should_receive(:load).and_raise(ArgumentError)
       lambda { job.payload_object }.should raise_error(Delayed::DeserializationError)
     end
@@ -169,7 +198,7 @@ shared_examples_for 'a delayed_job backend' do
 
     it "should reserve jobs scheduled for the past when time zones are involved" do
       Time.zone = 'US/Eastern'
-      job = create_job :run_at => described_class.db_time_now - 1.minute.ago.in_time_zone
+      job = create_job :run_at => described_class.db_time_now - 1.minute
       described_class.reserve(worker).should == job
     end
 
@@ -187,7 +216,7 @@ shared_examples_for 'a delayed_job backend' do
     end
 
     it "should reserve expired jobs" do
-      job = create_job(:locked_by => worker.name, :locked_at => described_class.db_time_now - 3.minutes)
+      job = create_job(:locked_by => 'some other worker', :locked_at => described_class.db_time_now - Delayed::Worker.max_run_time - 1.minute)
       described_class.reserve(worker).should == job
     end
 
@@ -215,8 +244,7 @@ shared_examples_for 'a delayed_job backend' do
     it "should parse from handler on deserialization error" do
       job = Story.create(:text => "...").delay.text
       job.payload_object.object.destroy
-      job = described_class.find(job.id)
-      job.name.should == 'Delayed::PerformableMethod'
+      job.reload.name.should == 'Delayed::PerformableMethod'
     end
   end
 
@@ -289,20 +317,73 @@ shared_examples_for 'a delayed_job backend' do
       @job.id.should_not be_nil
     end
   end
-  
+
+  context "named queues" do
+    context "when worker has one queue set" do
+      before(:each) do
+        worker.queues = ['large']
+      end
+
+      it "should only work off jobs which are from its queue" do
+        SimpleJob.runs.should == 0
+
+        create_job(:queue => "large")
+        create_job(:queue => "small")
+        worker.work_off
+
+        SimpleJob.runs.should == 1
+      end
+    end
+
+    context "when worker has two queue set" do
+      before(:each) do
+        worker.queues = ['large', 'small']
+      end
+
+      it "should only work off jobs which are from its queue" do
+        SimpleJob.runs.should == 0
+
+        create_job(:queue => "large")
+        create_job(:queue => "small")
+        create_job(:queue => "medium")
+        create_job
+        worker.work_off
+
+        SimpleJob.runs.should == 2
+      end
+    end
+
+    context "when worker does not have queue set" do
+      before(:each) do
+        worker.queues = []
+      end
+
+      it "should work off all jobs" do
+        SimpleJob.runs.should == 0
+
+        create_job(:queue => "one")
+        create_job(:queue => "two")
+        create_job
+        worker.work_off
+
+        SimpleJob.runs.should == 3
+      end
+    end
+  end
+
   context "max_attempts" do
     before(:each) do
       @job = described_class.enqueue SimpleJob.new
     end
-    
+
     it 'should not be defined' do
       @job.max_attempts.should be_nil
     end
-    
+
     it 'should use the max_retries value on the payload when defined' do
       @job.payload_object.stub!(:max_attempts).and_return(99)
       @job.max_attempts.should == 99
-    end 
+    end
   end
 
   describe "yaml serialization" do
@@ -310,7 +391,7 @@ shared_examples_for 'a delayed_job backend' do
       story = Story.create(:text => 'hello')
       job = story.delay.tell
       story.update_attributes :text => 'goodbye'
-      described_class.find(job.id).payload_object.object.text.should == 'goodbye'
+      job.reload.payload_object.object.text.should == 'goodbye'
     end
 
     it "should raise deserialization error for destroyed records" do
@@ -318,7 +399,7 @@ shared_examples_for 'a delayed_job backend' do
       job = story.delay.tell
       story.destroy
       lambda {
-        described_class.find(job.id).payload_object
+        job.reload.payload_object
       }.should raise_error(Delayed::DeserializationError)
     end
   end

data/lib/delayed/command.rb

@@ -5,17 +5,16 @@ require 'optparse'
 module Delayed
   class Command
     attr_accessor :worker_count
-    
+
     def initialize(args)
-      @files_to_reopen = []
       @options = {
         :quiet => true,
         :pid_dir => "#{Rails.root}/tmp/pids"
       }
-      
+
       @worker_count = 1
       @monitor = false
-      
+
       opts = OptionParser.new do |opts|
         opts.banner = "Usage: #{File.basename($0)} [options] start|stop|restart|run"
 
@@ -50,20 +49,20 @@ module Delayed
         opts.on('-p', '--prefix NAME', "String to be prefixed to worker process names") do |prefix|
           @options[:prefix] = prefix
         end
+        opts.on('--queues=queues', "Specify which queue DJ must look up for jobs") do |queues|
+          @options[:queues] = queues.split(',')
+        end
+        opts.on('--queue=queue', "Specify which queue DJ must look up for jobs") do |queue|
+          @options[:queues] = queue.split(',')
+        end
       end
       @args = opts.parse!(args)
     end
-  
-    def daemonize
-      Delayed::Worker.backend.before_fork
 
-      ObjectSpace.each_object(File) do |file|
-        @files_to_reopen << file unless file.closed?
-      end
-      
+    def daemonize
       dir = @options[:pid_dir]
       Dir.mkdir(dir) unless File.exists?(dir)
-      
+
       if @worker_count > 1 && @options[:identifier]
         raise ArgumentError, 'Cannot specify both --number-of-workers and --identifier'
       elsif @worker_count == 1 && @options[:identifier]
@@ -76,29 +75,21 @@ module Delayed
         end
       end
     end
-    
+
     def run_process(process_name, dir)
+      Delayed::Worker.before_fork
       Daemons.run_proc(process_name, :dir => dir, :dir_mode => :normal, :monitor => @monitor, :ARGV => @args) do |*args|
         $0 = File.join(@options[:prefix], process_name) if @options[:prefix]
         run process_name
       end
     end
-    
+
     def run(worker_name = nil)
       Dir.chdir(Rails.root)
-      
-      # Re-open file handles
-      @files_to_reopen.each do |file|
-        begin
-          file.reopen file.path, "a+"
-          file.sync = true
-        rescue ::Exception
-        end
-      end
-      
+
+      Delayed::Worker.after_fork
       Delayed::Worker.logger = Logger.new(File.join(Rails.root, 'log', 'delayed_job.log'))
-      Delayed::Worker.backend.after_fork
-      
+
       worker = Delayed::Worker.new(@options)
       worker.name_prefix = "#{worker_name} "
       worker.start
@@ -107,6 +98,5 @@ module Delayed
       STDERR.puts e.message
       exit 1
     end
-    
   end
 end

data/lib/delayed/lifecycle.rb

@@ -0,0 +1,84 @@
+module Delayed
+  class InvalidCallback < Exception; end
+  
+  class Lifecycle
+    EVENTS = {
+      :enqueue => [:job], 
+      :execute => [:worker], 
+      :loop => [:worker], 
+      :perform => [:worker, :job], 
+      :error => [:worker, :job], 
+      :failure => [:worker, :job], 
+      :invoke_job => [:job]
+    }
+    
+    def initialize
+      @callbacks = EVENTS.keys.inject({}) { |hash, e| hash[e] = Callback.new; hash }
+    end
+    
+    def before(event, &block)
+      add(:before, event, &block)
+    end
+    
+    def after(event, &block)
+      add(:after, event, &block)
+    end
+    
+    def around(event, &block)
+      add(:around, event, &block)
+    end
+    
+    def run_callbacks(event, *args, &block)
+      missing_callback(event) unless @callbacks.has_key?(event)
+      
+      unless EVENTS[event].size == args.size
+        raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(', ')}"
+      end
+      
+      @callbacks[event].execute(*args, &block)
+    end
+    
+    private
+    
+      def add(type, event, &block)
+        missing_callback(event) unless @callbacks.has_key?(event)
+
+        @callbacks[event].add(type, &block)
+      end
+      
+      def missing_callback(event)
+        raise InvalidCallback, "Unknown callback event: #{event}"
+      end
+  end
+  
+  class Callback
+    def initialize
+      @before = []
+      @after = []
+      
+      # Identity proc. Avoids special cases when there is no existing around chain.
+      @around = lambda { |*args, &block| block.call(*args) }
+    end
+    
+    def execute(*args, &block)      
+      @before.each { |c| c.call(*args) }
+      result = @around.call(*args, &block)
+      @after.each { |c| c.call(*args) }
+      result
+    end
+    
+    def add(type, &callback)      
+      case type
+      when :before
+        @before << callback
+      when :after
+        @after << callback
+      when :around
+        chain = @around # use a local variable so that the current chain is closed over in the following lambda
+        @around = lambda { |*a, &block| chain.call(*a) { |*b| callback.call(*b, &block) } }
+      else
+        raise InvalidCallback, "Invalid callback type: #{type}"
+      end
+    end
+  end
+end

data/lib/delayed/performable_method.rb

@@ -1,7 +1,9 @@
 require 'active_support/core_ext/module/delegation'
 
 module Delayed
-  class PerformableMethod < Struct.new(:object, :method_name, :args)
+  class PerformableMethod
+    attr_accessor :object, :method_name, :args
+
     delegate :method, :to => :object
 
     def initialize(object, method_name, args)

data/lib/delayed/plugin.rb

@@ -0,0 +1,13 @@
+module Delayed
+  class Plugin
+    class_attribute :callback_block
+
+    def self.callbacks(&block)
+      self.callback_block = block
+    end
+
+    def initialize
+      self.class.callback_block.call(Delayed::Worker.lifecycle) if self.class.callback_block
+    end
+  end
+end

data/lib/delayed/plugins/clear_locks.rb

@@ -0,0 +1,15 @@
+module Delayed
+  module Plugins
+    class ClearLocks < Plugin
+      callbacks do |lifecycle|
+        lifecycle.around(:execute) do |worker, &block|
+          begin
+            block.call(worker)
+          ensure            
+            Delayed::Job.clear_locks!(worker.name)
+          end
+        end 
+      end
+    end
+  end
+end

data/lib/delayed/psych_ext.rb

@@ -0,0 +1,65 @@
+class ActiveRecord::Base
+  # serialize to YAML
+  def encode_with(coder)
+    coder["attributes"] = @attributes
+    coder.tag = ['!ruby/ActiveRecord', self.class.name].join(':')
+  end
+end
+
+class Delayed::PerformableMethod
+  # serialize to YAML
+  def encode_with(coder)
+    coder.map = {
+      "object" => object,
+      "method_name" => method_name,
+      "args" => args
+    }
+  end
+end
+
+module Psych
+  module Visitors
+    class YAMLTree
+      def visit_Class(klass)
+        tag = ['!ruby/class', klass.name].join(':')
+        register(klass, @emitter.start_mapping(nil, tag, false, Nodes::Mapping::BLOCK))
+        @emitter.end_mapping
+      end
+    end
+
+    class ToRuby
+      def visit_Psych_Nodes_Mapping_with_class(object)
+        return revive(Psych.load_tags[object.tag], object) if Psych.load_tags[object.tag]
+
+        case object.tag
+        when /^!ruby\/class:?(.*)?$/
+          resolve_class $1
+        when /^!ruby\/ActiveRecord:(.+)$/
+          klass = resolve_class($1)
+          payload = Hash[*object.children.map { |c| accept c }]
+          id = payload["attributes"][klass.primary_key]
+          begin
+            if ActiveRecord::VERSION::MAJOR == 3
+              klass.unscoped.find(id)
+            else # Rails 2
+              klass.with_exclusive_scope { klass.find(id) }
+            end
+          rescue ActiveRecord::RecordNotFound
+            raise Delayed::DeserializationError
+          end
+        else
+          visit_Psych_Nodes_Mapping_without_class(object)
+        end
+      end
+      alias_method_chain :visit_Psych_Nodes_Mapping, :class
+
+      def resolve_class_with_constantize(klass_name)
+        klass_name.constantize
+      rescue
+        resolve_class_without_constantize(klass_name)
+      end
+      alias_method_chain :resolve_class, :constantize
+    end
+  end
+end
+

data/lib/delayed/railtie.rb

@@ -4,8 +4,6 @@ require 'rails'
 module Delayed
   class Railtie < Rails::Railtie
     initializer :after_initialize do
-      Delayed::Worker.guess_backend
-
       ActiveSupport.on_load(:action_mailer) do
         ActionMailer::Base.send(:extend, Delayed::DelayMail)
       end

data/lib/delayed/serialization/active_record.rb

@@ -2,7 +2,11 @@ class ActiveRecord::Base
   yaml_as "tag:ruby.yaml.org,2002:ActiveRecord"
 
   def self.yaml_new(klass, tag, val)
-    klass.find(val['attributes']['id'])
+    if ActiveRecord::VERSION::MAJOR == 3
+      klass.unscoped.find(val['attributes'][klass.primary_key])
+    else # Rails 2
+      klass.with_exclusive_scope { klass.find(val['attributes'][klass.primary_key]) }
+    end
   rescue ActiveRecord::RecordNotFound
     raise Delayed::DeserializationError
   end

data/lib/delayed/syck_ext.rb

@@ -0,0 +1,34 @@
+class Module
+  yaml_as "tag:ruby.yaml.org,2002:module"
+
+  def self.yaml_new(klass, tag, val)
+    klass
+  end
+
+  def to_yaml(options = {})
+    YAML.quick_emit(nil, options) do |out|
+      out.scalar(taguri, name, :plain)
+    end
+  end
+
+  def yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    name
+  end
+end
+
+class Class
+  yaml_as "tag:ruby.yaml.org,2002:class"
+  remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
+end
+
+class Struct
+  def self.yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    "Struct::#{ name }"
+  end
+end

data/lib/delayed/tasks.rb

@@ -6,6 +6,6 @@ namespace :jobs do
 
   desc "Start a delayed_job worker."
   task :work => :environment do
-    Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY'], :quiet => false).start
+    Delayed::Worker.new(:min_priority => ENV['MIN_PRIORITY'], :max_priority => ENV['MAX_PRIORITY'], :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','), :quiet => false).start
   end
 end