jeffkreeftmeijer-delayed_job 0.1.0

data/lib/delayed/job/mongo_job.rb

@@ -0,0 +1,185 @@
+module Delayed
+
+  # A job object that is persisted to the database.
+  # Contains the work object as a YAML field.
+  class Job
+    include MongoMapper::Document
+
+    key :priority,  Integer, :default => 0
+    key :attempts,  Integer, :default => 0
+    key :handler
+    key :last_error
+    key :run_at,    Time
+    key :locked_at, Time
+    key :locked_by
+    key :failed_at, Time, :default => nil
+    timestamps!
+
+    collection_name = 'delayed_jobs'
+
+    def self.last(opts={})
+      super(opts.merge(:order => 'id'))
+    end
+
+    # When a worker is exiting, make sure we don't have any locked jobs.
+    def self.clear_locks!
+      collection.update({:locked_by => worker_name}, {"$set" => {:locked_by => nil, :locked_at => nil}}, :multi => true)
+    end
+
+    # Reschedule the job in the future (when a job fails).
+    # Uses an exponential scale depending on the number of failed attempts.
+    def reschedule(message, backtrace = [], time = nil)
+      if self.attempts < MAX_ATTEMPTS
+        time ||= Job.db_time_now + (attempts ** 4) + 5
+
+        self.attempts    += 1
+        self.run_at       = time
+        self.last_error   = message + "\n" + backtrace.join("\n")
+        self.unlock
+        save!
+      else
+        logger.info "* [JOB] PERMANENTLY removing #{self.name} because of #{attempts} consequetive failures."
+        destroy_failed_jobs ? destroy : update_attributes(:failed_at => Delayed::Job.db_time_now)
+      end
+    end
+
+    # Try to run one job. Returns true/false (work done/work failed) or nil if job can't be locked.
+    def run_with_lock(max_run_time, worker_name)
+      logger.info "* [JOB] aquiring lock on #{name}"
+      unless lock_exclusively!(max_run_time, worker_name)
+        # We did not get the lock, some other worker process must have
+        logger.warn "* [JOB] failed to aquire exclusive lock for #{name}"
+        return nil # no work done
+      end
+
+      begin
+        runtime =  Benchmark.realtime do
+          invoke_job # TODO: raise error if takes longer than max_run_time
+          logger.info "Also destroying self"
+          destroy
+        end
+        # TODO: warn if runtime > max_run_time ?
+        logger.info "* [JOB] #{name} completed after %.4f" % runtime
+        return true  # did work
+      rescue Exception => e
+        reschedule e.message, e.backtrace
+        log_exception(e)
+        return false  # work failed
+      end
+    end
+
+    # Add a job to the queue
+    def self.enqueue(*args, &block)
+      object = block_given? ? EvaledJob.new(&block) : args.shift
+
+      unless object.respond_to?(:perform) || block_given?
+        raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
+      end
+
+      priority = args.first || 0
+      run_at   = args[1]
+
+      Job.create(:payload_object => object, :priority => priority.to_i, :run_at => run_at)
+    end
+
+    # Find a few candidate jobs to run (in case some immediately get locked by others).
+    # Return in random order prevent everyone trying to do same head job at once.
+    def self.find_available(limit = 5, max_run_time = MAX_RUN_TIME)
+      time_now = db_time_now
+
+      conditions = {}
+
+      if self.min_priority
+        conditions["priority"] ||= {}
+        conditions["priority"].merge!("$gte" => min_priority)
+      end
+
+      if self.max_priority
+        conditions["priority"] ||= {}
+        conditions["priority"].merge!("$lte" => max_priority)
+      end
+
+      overtime = make_date(time_now - max_run_time.to_i)
+      query = "(this.run_at <= #{make_date(time_now)} && (this.locked_at == null || this.locked_at < #{overtime}) || this.locked_by == '#{worker_name}') && this.failed_at == null"
+
+      conditions.merge!("$where" => make_query(query))
+
+      records = collection.find(conditions, {:sort => [['priority', 'descending'], ['run_at', 'ascending']], :limit => limit}).map {|x| new(x)} #, :order => NextTaskOrder, :limit => limit)
+      records.sort_by { rand() }
+    end
+
+    def self.make_date(date)
+      "new Date(#{date.to_f * 1000})"
+    end
+
+    def make_date(date)
+      self.class.make_date(date)
+    end
+
+    def self.make_query(string)
+      Mongo::Code.new("function() { return (#{string}); }")
+    end
+
+    def make_query(string)
+      self.class.make_query(string)
+    end
+
+    # Run the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def self.reserve_and_run_one_job(max_run_time = MAX_RUN_TIME)
+
+      # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+      # this leads to a more even distribution of jobs across the worker processes
+      find_available(5, max_run_time).each do |job|
+        t = job.run_with_lock(max_run_time, worker_name)
+        return t unless t == nil  # return if we did work (good or bad)
+      end
+
+      nil # we didn't do any work, all 5 were not lockable
+    end
+
+    # Lock this job for this worker.
+    # Returns true if we have the lock, false otherwise.
+    def lock_exclusively!(max_run_time, worker = worker_name)
+      now = self.class.db_time_now
+
+      affected_rows = if locked_by != worker
+        overtime = make_date(now - max_run_time.to_i)
+        query = "this._id == '#{id}' && (this.locked_at == null || this.locked_at < #{overtime})"
+
+        conditions = {"$where" => make_query(query)}
+        matches = collection.find(conditions).count
+        collection.update(conditions, {"$set" => {:locked_at => now, :locked_by => worker}}, :multi => true)
+        matches
+      else
+        conditions = {"_id" => Mongo::ObjectID.from_string(id), "locked_by" => worker} 
+        matches = collection.find(conditions).count
+        collection.update(conditions, {"$set" => {"locked_at" => now}}, :multi => true)
+        matches
+      end
+      if affected_rows == 1
+        self.locked_at    = now
+        self.locked_by    = worker
+        return true
+      else
+        return false
+      end
+    end
+
+  private
+
+    # Get the current time (GMT or local depending on DB)
+    # Note: This does not ping the DB to get the time, so all your clients
+    # must have syncronized clocks.
+    def self.db_time_now
+      Time.now.utc
+    end
+
+  protected
+
+    def before_save
+      self.run_at ||= self.class.db_time_now
+    end
+
+  end
+end

data/lib/delayed/message_sending.rb

@@ -0,0 +1,17 @@
+module Delayed
+  module MessageSending
+    def send_later(method, *args)
+      Delayed::Job.enqueue Delayed::PerformableMethod.new(self, method.to_sym, args)
+    end
+    
+    module ClassMethods
+      def handle_asynchronously(method)
+        without_name = "#{method}_without_send_later"
+        define_method("#{method}_with_send_later") do |*args|
+          send_later(without_name, *args)
+        end
+        alias_method_chain method, :send_later
+      end
+    end
+  end                               
+end

data/lib/delayed/performable_method.rb

@@ -0,0 +1,55 @@
+module Delayed
+  class PerformableMethod < Struct.new(:object, :method, :args)
+    CLASS_STRING_FORMAT = /^CLASS\:([A-Z][\w\:]+)$/
+    AR_STRING_FORMAT    = /^AR\:([A-Z][\w\:]+)\:(\d+)$/
+
+    def initialize(object, method, args)
+      raise NoMethodError, "undefined method `#{method}' for #{self.inspect}" unless object.respond_to?(method)
+
+      self.object = dump(object)
+      self.args   = args.map { |a| dump(a) }
+      self.method = method.to_sym
+    end
+    
+    def display_name  
+      case self.object
+      when CLASS_STRING_FORMAT then "#{$1}.#{method}"
+      when AR_STRING_FORMAT    then "#{$1}##{method}"
+      else "Unknown##{method}"
+      end      
+    end    
+
+    def perform
+      load(object).send(method, *args.map{|a| load(a)})
+    rescue PERFORMABLE_METHOD_EXCEPTION
+      # We cannot do anything about objects which were deleted in the meantime
+      true
+    end
+
+    private
+
+    def load(arg)
+      case arg
+      when CLASS_STRING_FORMAT then $1.constantize
+      when AR_STRING_FORMAT    then $1.constantize.find($2)
+      else arg
+      end
+    end
+
+    def dump(arg)
+      case arg
+      when Class              then class_to_string(arg)
+      when PERFORMABLE_METHOD_STORE then ar_to_string(arg)
+      else arg
+      end
+    end
+
+    def ar_to_string(obj)
+      "AR:#{obj.class}:#{obj.id}"
+    end
+
+    def class_to_string(obj)
+      "CLASS:#{obj.name}"
+    end
+  end
+end

data/lib/delayed/worker.rb

@@ -0,0 +1,54 @@
+module Delayed
+  class Worker
+    SLEEP = 5
+
+    cattr_accessor :logger
+    self.logger = if defined?(Merb::Logger)
+      Merb.logger
+    elsif defined?(RAILS_DEFAULT_LOGGER)
+      RAILS_DEFAULT_LOGGER
+    end
+
+    def initialize(options={})
+      @quiet = options[:quiet]
+      Delayed::Job.min_priority = options[:min_priority] if options.has_key?(:min_priority)
+      Delayed::Job.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+    end
+
+    def start
+      say "*** Starting job worker #{Delayed::Job.worker_name}"
+
+      trap('TERM') { say 'Exiting...'; $exit = true }
+      trap('INT')  { say 'Exiting...'; $exit = true }
+
+      loop do
+        result = nil
+
+        realtime = Benchmark.realtime do
+          result = Delayed::Job.work_off
+        end
+
+        count = result.sum
+
+        break if $exit
+
+        if count.zero?
+          sleep(SLEEP)
+        else
+          say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+        end
+
+        break if $exit
+      end
+
+    ensure
+      Delayed::Job.clear_locks!
+    end
+
+    def say(text)
+      puts text unless @quiet
+      logger.info text if logger
+    end
+
+  end
+end

data/lib/delayed_job.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/delayed/message_sending'
+require File.dirname(__FILE__) + '/delayed/performable_method'
+
+if defined?(MongoMapper)
+  require File.dirname(__FILE__) + '/delayed/job/mongo_job'
+
+  PERFORMABLE_METHOD_EXCEPTION = Exception
+  PERFORMABLE_METHOD_STORE = MongoMapper::Document
+else
+  autoload :ActiveRecord, 'activerecord'
+  require File.dirname(__FILE__) + '/delayed/job/active_record_job'
+
+  PERFORMABLE_METHOD_EXCEPTION = ActiveRecord::RecordNotFound
+  PERFORMABLE_METHOD_STORE = ActiveRecord::Base
+end
+
+require File.dirname(__FILE__) + '/delayed/job'
+require File.dirname(__FILE__) + '/delayed/worker'
+
+Object.send(:include, Delayed::MessageSending)   
+Module.send(:include, Delayed::MessageSending::ClassMethods)  
+
+if defined?(Merb::Plugins)
+  Merb::Plugins.add_rakefiles File.dirname(__FILE__) / '..' / 'tasks' / 'tasks'
+end

data/spec/delayed_method_spec.rb

@@ -0,0 +1,126 @@
+class SimpleJob
+  cattr_accessor :runs; self.runs = 0
+  def perform; @@runs += 1; end
+end
+
+class RandomRubyObject  
+  def say_hello
+    'hello'
+  end
+end
+
+class ErrorObject
+
+  def throw
+    raise ActiveRecord::RecordNotFound, '...'
+    false
+  end
+
+end
+
+class StoryReader
+
+  def read(story)
+    "Epilog: #{story.tell}"
+  end
+
+end
+
+class StoryReader
+
+  def read(story)
+    "Epilog: #{story.tell}"
+  end
+
+end
+
+describe 'random ruby objects' do
+  before       { Delayed::Job.delete_all }
+
+  it "should respond_to :send_later method" do
+
+    RandomRubyObject.new.respond_to?(:send_later)
+
+  end
+
+  it "should raise a ArgumentError if send_later is called but the target method doesn't exist" do
+    lambda { RandomRubyObject.new.send_later(:method_that_deos_not_exist) }.should raise_error(NoMethodError)
+  end
+
+  it "should add a new entry to the job table when send_later is called on it" do
+    Delayed::Job.count.should == 0
+
+    RandomRubyObject.new.send_later(:to_s)
+
+    Delayed::Job.count.should == 1
+  end
+
+  it "should add a new entry to the job table when send_later is called on the class" do
+    Delayed::Job.count.should == 0
+
+    RandomRubyObject.send_later(:to_s)
+
+    Delayed::Job.count.should == 1
+  end
+
+  it "should run get the original method executed when the job is performed" do
+
+    RandomRubyObject.new.send_later(:say_hello)
+
+    Delayed::Job.count.should == 1
+  end
+
+  it "should ignore ActiveRecord::RecordNotFound errors because they are permanent" do
+
+    ErrorObject.new.send_later(:throw)
+
+    Delayed::Job.count.should == 1
+
+    Delayed::Job.reserve_and_run_one_job
+
+    Delayed::Job.count.should == 0
+
+  end
+
+  it "should store the object as string if its an active record" do
+    story = Story.create :text => 'Once upon...'
+    story.send_later(:tell)
+
+    job =  Delayed::Job.find(:first)
+    job.payload_object.class.should   == Delayed::PerformableMethod
+    job.payload_object.object.should  == "AR:Story:#{story.id}"
+    job.payload_object.method.should  == :tell
+    job.payload_object.args.should    == []
+    job.payload_object.perform.should == 'Once upon...'
+  end
+
+  it "should store arguments as string if they an active record" do
+
+    story = Story.create :text => 'Once upon...'
+
+    reader = StoryReader.new
+    reader.send_later(:read, story)
+
+    job =  Delayed::Job.find(:first)
+    job.payload_object.class.should   == Delayed::PerformableMethod
+    job.payload_object.method.should  == :read
+    job.payload_object.args.should    == ["AR:Story:#{story.id}"]
+    job.payload_object.perform.should == 'Epilog: Once upon...'
+  end                 
+  
+  it "should call send later on methods which are wrapped with handle_asynchronously" do
+    story = Story.create :text => 'Once upon...'
+  
+    Delayed::Job.count.should == 0
+  
+    story.whatever(1, 5)
+  
+    Delayed::Job.count.should == 1
+    job =  Delayed::Job.find(:first)
+    job.payload_object.class.should   == Delayed::PerformableMethod
+    job.payload_object.method.should  == :whatever_without_send_later
+    job.payload_object.args.should    == [1, 5]
+    job.payload_object.perform.should == 'Once upon...'
+  end
+
+end