topprospect-delayed_job 2.0.5

data/lib/delayed/worker.rb

@@ -0,0 +1,213 @@
+require 'timeout'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/core_ext/kernel'
+
+module Delayed
+  DEFAULT_QUEUE = ''
+
+  class Worker
+    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger, :queue
+    self.sleep_delay = 5
+    self.max_attempts = 25
+    self.max_run_time = 4.hours
+    self.default_priority = 0
+    self.queue = Delayed::DEFAULT_QUEUE
+    
+    # By default failed jobs are destroyed after too many attempts. If you want to keep them around
+    # (perhaps to inspect the reason for the failure), set this to false.
+    cattr_accessor :destroy_failed_jobs
+    self.destroy_failed_jobs = true
+    
+    self.logger = if defined?(Merb::Logger)
+      Merb.logger
+    elsif defined?(RAILS_DEFAULT_LOGGER)
+      RAILS_DEFAULT_LOGGER
+    end
+
+    # name_prefix is ignored if name is set directly
+    attr_accessor :name_prefix
+    
+    cattr_reader :backend
+
+    # rename the default Rails logger file, if requested
+    # TODO should we put it inside a Rails own class/module?
+    #
+    # http://stackoverflow.com/questions/3500200/getting-delayed-job-to-log
+    # https://gist.github.com/833828
+
+    def rename_default_rails_log_if_given(filename)
+      return unless filename and not filename.empty?
+
+      f = open filename, (File::WRONLY | File::APPEND | File::CREAT)
+      f.sync = true
+      RAILS_DEFAULT_LOGGER.auto_flushing = true
+      # TODO shouldn't we first close whatever was there?
+      RAILS_DEFAULT_LOGGER.instance_variable_set(:@log, f)
+    end
+    
+    def self.backend=(backend)
+      if backend.is_a? Symbol
+        require "delayed/backend/#{backend}"
+        backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
+      end
+      @@backend = backend
+      silence_warnings { ::Delayed.const_set(:Job, backend) }
+    end
+    
+    def self.guess_backend
+      self.backend ||= if defined?(ActiveRecord)
+        :active_record
+      elsif defined?(MongoMapper)
+        :mongo_mapper
+      else
+        logger.warn "Could not decide on a backend, defaulting to active_record"
+        :active_record
+      end
+    end
+
+    def initialize(options={})
+      @quiet = options[:quiet]
+      self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
+      self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+      self.class.queue        = options[:queue]  || Delayed::DEFAULT_QUEUE
+      # renaming the log file works like this only when using RAILS_DEFAULT_LOGGER:
+      rename_default_rails_log_if_given(options[:logname])
+    end
+
+    # Every worker has a unique name which by default is the pid of the process. There are some
+    # advantages to overriding this with something which survives worker retarts:  Workers can#
+    # safely resume working on tasks which are locked by themselves. The worker will assume that
+    # it crashed before.
+    def name
+      return @name unless @name.nil?
+      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
+    end
+
+    # Sets the name of the worker.
+    # Setting the name to nil will reset the default worker name
+    def name=(val)
+      @name = val
+    end
+
+    def start
+      say "Starting job worker"
+
+      trap('TERM') { say 'Exiting...'; $exit = true }
+      trap('INT')  { say 'Exiting...'; $exit = true }
+
+      loop do
+        result = nil
+
+        realtime = Benchmark.realtime do
+          result = work_off
+        end
+
+        count = result.sum
+
+        break if $exit
+
+        if count.zero?
+          sleep(@@sleep_delay)
+        else
+          say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+        end
+
+        break if $exit
+      end
+
+    ensure
+      Delayed::Job.clear_locks!(name)
+    end
+    
+    # Do num jobs and return stats on success/failure.
+    # Exit early if interrupted.
+    def work_off(num = 100)
+      success, failure = 0, 0
+
+      num.times do
+        case reserve_and_run_one_job
+        when true
+            success += 1
+        when false
+            failure += 1
+        else
+          break  # leave if no work could be done
+        end
+        break if $exit # leave if we're exiting
+      end
+
+      return [success, failure]
+    end
+    
+    def run(job)
+      runtime =  Benchmark.realtime do
+        Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
+        job.destroy
+      end
+      say "#{job.name} completed after %.4f" % runtime
+      return true  # did work
+    rescue Exception => e
+      handle_failed_job(job, e)
+      return false  # work failed
+    end
+    
+    # Reschedule the job in the future (when a job fails).
+    # Uses an exponential scale depending on the number of failed attempts.
+    def reschedule(job, time = nil)
+      if (job.attempts += 1) < self.class.max_attempts
+        time ||= Job.db_time_now + (job.attempts ** 4) + 5
+        job.run_at = time
+        job.unlock
+        job.save!
+      else
+        say "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+
+        if job.payload_object.respond_to? :on_permanent_failure
+          say "Running on_permanent_failure hook"
+          failure_method = job.payload_object.method(:on_permanent_failure)
+          if failure_method.arity == 1
+            failure_method.call(job)
+          else
+            failure_method.call
+          end
+        end
+
+        self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
+      end
+    end
+
+    def say(text, level = Logger::INFO)
+      text = "[Worker(#{name})] #{text}"
+      puts text unless @quiet
+      logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
+    end
+
+  protected
+    
+    def handle_failed_job(job, error)
+      job.last_error = error.message + "\n" + error.backtrace.join("\n")
+      say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
+      reschedule(job)
+    end
+    
+    # Run the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def reserve_and_run_one_job
+
+      # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+      # this leads to a more even distribution of jobs across the worker processes
+      job = Delayed::Job.find_available(name, 5, self.class.max_run_time).detect do |job|
+        if job.lock_exclusively!(self.class.max_run_time, name)
+          say "acquired lock on #{job.name}"
+          true
+        else
+          say "failed to acquire exclusive lock for #{job.name}", Logger::WARN
+          false
+        end
+      end
+
+      run(job) if job
+    end
+  end
+end

data/lib/delayed/worker.rb.orig

@@ -0,0 +1,202 @@
+require 'timeout'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/core_ext/kernel'
+
+module Delayed
+  DEFAULT_QUEUE = ''
+
+  class Worker
+    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time, :default_priority, :sleep_delay, :logger, :queue
+    self.sleep_delay = 5
+    self.max_attempts = 25
+    self.max_run_time = 4.hours
+    self.default_priority = 0
+    self.queue = Delayed::DEFAULT_QUEUE
+    
+    # By default failed jobs are destroyed after too many attempts. If you want to keep them around
+    # (perhaps to inspect the reason for the failure), set this to false.
+    cattr_accessor :destroy_failed_jobs
+    self.destroy_failed_jobs = true
+    
+    self.logger = if defined?(Merb::Logger)
+      Merb.logger
+    elsif defined?(RAILS_DEFAULT_LOGGER)
+      RAILS_DEFAULT_LOGGER
+    end
+
+    # name_prefix is ignored if name is set directly
+    attr_accessor :name_prefix
+    
+    cattr_reader :backend
+    
+    def self.backend=(backend)
+      if backend.is_a? Symbol
+        require "delayed/backend/#{backend}"
+        backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
+      end
+      @@backend = backend
+      silence_warnings { ::Delayed.const_set(:Job, backend) }
+    end
+    
+    def self.guess_backend
+      self.backend ||= if defined?(ActiveRecord)
+        :active_record
+      elsif defined?(MongoMapper)
+        :mongo_mapper
+      else
+        logger.warn "Could not decide on a backend, defaulting to active_record"
+        :active_record
+      end
+    end
+
+    def initialize(options={})
+      @quiet = options[:quiet]
+      self.class.min_priority = options[:min_priority] if options.has_key?(:min_priority)
+      self.class.max_priority = options[:max_priority] if options.has_key?(:max_priority)
+<<<<<<< HEAD
+      self.class.queue        = options[:queue]  || Delayed::DEFAULT_QUEUE
+      # renaming the log file works like this only when using RAILS_DEFAULT_LOGGER:
+      rename_default_rails_log_if_given(options[:logname])
+=======
+      self.class.sleep_delay = options[:sleep_delay] if options.has_key?(:sleep_delay)
+      self.class.queue        = options[:queue] || Delayed::DEFAULT_QUEUE
+>>>>>>> 75b6e0c... added queues from the original topprospect patch, added tests for queues
+    end
+
+    # Every worker has a unique name which by default is the pid of the process. There are some
+    # advantages to overriding this with something which survives worker retarts:  Workers can#
+    # safely resume working on tasks which are locked by themselves. The worker will assume that
+    # it crashed before.
+    def name
+      return @name unless @name.nil?
+      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
+    end
+
+    # Sets the name of the worker.
+    # Setting the name to nil will reset the default worker name
+    def name=(val)
+      @name = val
+    end
+
+    def start
+      say "Starting job worker"
+
+      trap('TERM') { say 'Exiting...'; $exit = true }
+      trap('INT')  { say 'Exiting...'; $exit = true }
+
+      loop do
+        result = nil
+
+        realtime = Benchmark.realtime do
+          result = work_off
+        end
+
+        count = result.sum
+
+        break if $exit
+
+        if count.zero?
+          sleep(@@sleep_delay)
+        else
+          say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+        end
+
+        break if $exit
+      end
+
+    ensure
+      Delayed::Job.clear_locks!(name)
+    end
+    
+    # Do num jobs and return stats on success/failure.
+    # Exit early if interrupted.
+    def work_off(num = 100)
+      success, failure = 0, 0
+
+      num.times do
+        case reserve_and_run_one_job
+        when true
+            success += 1
+        when false
+            failure += 1
+        else
+          break  # leave if no work could be done
+        end
+        break if $exit # leave if we're exiting
+      end
+
+      return [success, failure]
+    end
+    
+    def run(job)
+      runtime =  Benchmark.realtime do
+        Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
+        job.destroy
+      end
+      say "#{job.name} completed after %.4f" % runtime
+      return true  # did work
+    rescue Exception => e
+      handle_failed_job(job, e)
+      return false  # work failed
+    end
+    
+    # Reschedule the job in the future (when a job fails).
+    # Uses an exponential scale depending on the number of failed attempts.
+    def reschedule(job, time = nil)
+      if (job.attempts += 1) < self.class.max_attempts
+        time ||= Job.db_time_now + (job.attempts ** 4) + 5
+        job.run_at = time
+        job.unlock
+        job.save!
+      else
+        say "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+
+        if job.payload_object.respond_to? :on_permanent_failure
+          say "Running on_permanent_failure hook"
+          failure_method = job.payload_object.method(:on_permanent_failure)
+          if failure_method.arity == 1
+            failure_method.call(job)
+          else
+            failure_method.call
+          end
+        end
+
+        self.class.destroy_failed_jobs ? job.destroy : job.update_attributes(:failed_at => Delayed::Job.db_time_now)
+      end
+    end
+
+    def say(text, level = Logger::INFO)
+      text = "[Worker(#{name})] #{text}"
+      puts text unless @quiet
+      logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
+    end
+
+  protected
+    
+    def handle_failed_job(job, error)
+      job.last_error = error.message + "\n" + error.backtrace.join("\n")
+      say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
+      reschedule(job)
+    end
+    
+    # Run the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def reserve_and_run_one_job
+
+      # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+      # this leads to a more even distribution of jobs across the worker processes
+      job = Delayed::Job.find_available(name, 5, self.class.max_run_time).detect do |job|
+        if job.lock_exclusively!(self.class.max_run_time, name)
+          say "acquired lock on #{job.name}"
+          true
+        else
+          say "failed to acquire exclusive lock for #{job.name}", Logger::WARN
+          false
+        end
+      end
+
+      run(job) if job
+    end
+  end
+end

data/lib/delayed/yaml_ext.rb

@@ -0,0 +1,40 @@
+# These extensions allow properly serializing and autoloading of
+# Classes, Modules and Structs
+
+require 'yaml'
+
+class Module
+  yaml_as "tag:ruby.yaml.org,2002:module"
+
+  def self.yaml_new(klass, tag, val)
+    val.constantize
+  end
+
+  def to_yaml( opts = {} )
+    YAML::quick_emit( nil, opts ) { |out|
+      out.scalar(taguri, self.name, :plain)
+    }
+  end
+  
+  def yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    name
+  end
+  
+end
+
+class Class
+  yaml_as "tag:ruby.yaml.org,2002:class"
+  remove_method :to_yaml # use Module's to_yaml
+end
+
+class Struct
+  def self.yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    "Struct::#{ name }"
+  end
+end

data/lib/delayed_job.rb

@@ -0,0 +1,15 @@
+require 'active_support'
+
+require File.dirname(__FILE__) + '/delayed/message_sending'
+require File.dirname(__FILE__) + '/delayed/performable_method'
+require File.dirname(__FILE__) + '/delayed/yaml_ext'
+require File.dirname(__FILE__) + '/delayed/backend/base'
+require File.dirname(__FILE__) + '/delayed/worker'
+require File.dirname(__FILE__) + '/delayed/railtie' if defined?(::Rails::Railtie)
+
+Object.send(:include, Delayed::MessageSending)   
+Module.send(:include, Delayed::MessageSending::ClassMethods)
+
+if defined?(Merb::Plugins)
+  Merb::Plugins.add_rakefiles File.dirname(__FILE__) / 'delayed' / 'tasks'
+end

data/lib/generators/delayed_job/delayed_job_generator.rb

@@ -0,0 +1,34 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class DelayedJobGenerator < Rails::Generators::Base
+
+  include Rails::Generators::Migration
+  
+  def self.source_root
+     @source_root ||= File.join(File.dirname(__FILE__), 'templates')
+  end
+
+  # Implement the required interface for Rails::Generators::Migration.
+  #
+  def self.next_migration_number(dirname) #:nodoc:
+    next_migration_number = current_migration_number(dirname) + 1
+    if ActiveRecord::Base.timestamped_migrations
+      [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
+    else
+      "%.3d" % next_migration_number
+    end
+  end
+  
+  def create_script_file
+    template 'script', 'script/delayed_job'
+    chmod 'script/delayed_job', 0755
+  end
+  
+  def create_migration_file
+    if defined?(ActiveRecord)
+      migration_template 'migration.rb', 'db/migrate/create_delayed_jobs.rb'
+    end
+  end
+
+end

data/lib/generators/delayed_job/templates/migration.rb

@@ -0,0 +1,21 @@
+class CreateDelayedJobs < ActiveRecord::Migration
+  def self.up
+    create_table :delayed_jobs, :force => true do |table|
+      table.integer  :priority, :default => 0      # Allows some jobs to jump to the front of the queue
+      table.integer  :attempts, :default => 0      # Provides for retries, but still fail eventually.
+      table.text     :handler                      # YAML-encoded string of the object that will do work
+      table.text     :last_error                   # reason for last failure (See Note below)
+      table.datetime :run_at                       # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :locked_at                    # Set when a client is working on this object
+      table.datetime :failed_at                    # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.string   :locked_by                    # Who is working on this object (if locked)
+      table.timestamps
+    end
+	
+    add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+  end
+  
+  def self.down
+    drop_table :delayed_jobs  
+  end
+end

data/lib/generators/delayed_job/templates/script

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+require 'delayed/command'
+Delayed::Command.new(ARGV).daemonize

data/recipes/delayed_job.rb

@@ -0,0 +1 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'delayed', 'recipes'))

data/spec/autoloaded/clazz.rb

@@ -0,0 +1,7 @@
+# Make sure this file does not get required manually
+module Autoloaded
+  class Clazz
+    def perform
+    end
+  end
+end

data/spec/autoloaded/struct.rb

@@ -0,0 +1,7 @@
+# Make sure this file does not get required manually
+module Autoloaded
+  class Struct < ::Struct.new(nil)
+    def perform
+    end
+  end
+end

data/spec/backend/active_record_job_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'backend/shared_backend_spec'
+require 'delayed/backend/active_record'
+
+describe Delayed::Backend::ActiveRecord::Job do
+  before(:all) do
+    @backend = Delayed::Backend::ActiveRecord::Job
+  end
+  
+  before(:each) do
+    Delayed::Backend::ActiveRecord::Job.delete_all
+    SimpleJob.runs = 0
+  end
+  
+  after do
+    Time.zone = nil
+  end
+  
+  it_should_behave_like 'a backend'
+    [Delayed::DEFAULT_QUEUE, "foo"].each do |queue|
+    context "when given a queue of #{queue}" do
+      before do
+        Delayed::Worker.queue = queue
+      end
+      it_should_behave_like 'a backend'
+    end
+  end
+
+  context "db_time_now" do
+    it "should return time in current time zone if set" do
+      Time.zone = 'Eastern Time (US & Canada)'
+      %w(EST EDT).should include(Delayed::Job.db_time_now.zone)
+    end
+    
+    it "should return UTC time if that is the AR default" do
+      Time.zone = nil
+      ActiveRecord::Base.default_timezone = :utc
+      Delayed::Backend::ActiveRecord::Job.db_time_now.zone.should == 'UTC'
+    end
+
+    it "should return local time if that is the AR default" do
+      Time.zone = 'Central Time (US & Canada)'
+      ActiveRecord::Base.default_timezone = :local
+      %w(CST CDT).should include(Delayed::Backend::ActiveRecord::Job.db_time_now.zone)
+    end
+  end
+  
+  describe "after_fork" do
+    it "should call reconnect on the connection" do
+      ActiveRecord::Base.connection.should_receive(:reconnect!)
+      Delayed::Backend::ActiveRecord::Job.after_fork
+    end
+  end
+end

data/spec/backend/couch_rest_job_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'backend/shared_backend_spec'
+require 'delayed/backend/couch_rest'
+
+describe Delayed::Backend::CouchRest::Job do
+  before(:all) do
+    @backend = Delayed::Backend::CouchRest::Job
+  end
+  
+  before(:each) do
+    @backend.delete_all
+  end
+  
+  it_should_behave_like 'a backend'
+end

data/spec/backend/data_mapper_job_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'backend/shared_backend_spec'
+require 'delayed/backend/data_mapper'
+
+describe Delayed::Backend::DataMapper::Job do
+  before(:all) do
+    @backend = Delayed::Backend::DataMapper::Job
+  end
+  
+  before(:each) do
+    # reset database before each example is run
+    DataMapper.auto_migrate!
+  end
+  
+  it_should_behave_like 'a backend'
+end