delayed_job 4.0.6 → 4.1.8

data/lib/delayed/command.rb

@@ -5,16 +5,21 @@ unless ENV['RAILS_ENV'] == 'test'
     raise "You need to add gem 'daemons' to your Gemfile if you wish to use it."
   end
 end
+require 'fileutils'
 require 'optparse'
+require 'pathname'
 
 module Delayed
   class Command # rubocop:disable ClassLength
     attr_accessor :worker_count, :worker_pools
 
+    DIR_PWD = Pathname.new Dir.pwd
+
     def initialize(args) # rubocop:disable MethodLength
       @options = {
         :quiet => true,
-        :pid_dir => "#{Rails.root}/tmp/pids"
+        :pid_dir => "#{root}/tmp/pids",
+        :log_dir => "#{root}/log"
       }
 
       @worker_count = 1
@@ -28,7 +33,7 @@ module Delayed
           exit 1
         end
         opt.on('-e', '--environment=NAME', 'Specifies the environment to run this delayed jobs under (test/development/production).') do |_e|
-          STDERR.puts 'The -e/--environment option has been deprecated and has no effect. Use RAILS_ENV and see http://github.com/collectiveidea/delayed_job/issues/#issue/7'
+          STDERR.puts 'The -e/--environment option has been deprecated and has no effect. Use RAILS_ENV and see http://github.com/collectiveidea/delayed_job/issues/7'
         end
         opt.on('--min-priority N', 'Minimum priority of jobs to run.') do |n|
           @options[:min_priority] = n
@@ -37,11 +42,14 @@ module Delayed
           @options[:max_priority] = n
         end
         opt.on('-n', '--number_of_workers=workers', 'Number of unique workers to spawn') do |worker_count|
-          @worker_count = worker_count.to_i rescue 1 # rubocop:disable RescueModifier
+          @worker_count = worker_count.to_i rescue 1
         end
         opt.on('--pid-dir=DIR', 'Specifies an alternate directory in which to store the process ids.') do |dir|
           @options[:pid_dir] = dir
         end
+        opt.on('--log-dir=DIR', 'Specifies an alternate directory in which to store the delayed_job log.') do |dir|
+          @options[:log_dir] = dir
+        end
         opt.on('-i', '--identifier=n', 'A numeric identifier for the worker.') do |n|
           @options[:identifier] = n
         end
@@ -69,22 +77,27 @@ module Delayed
         opt.on('--exit-on-complete', 'Exit when no more jobs are available to run. This will exit if all jobs are scheduled to run in the future.') do
           @options[:exit_on_complete] = true
         end
+        opt.on('--daemon-options a, b, c', Array, 'options to be passed through to daemons gem') do |daemon_options|
+          @daemon_options = daemon_options
+        end
       end
-      @args = opts.parse!(args)
+      @args = opts.parse!(args) + (@daemon_options || [])
     end
 
     def daemonize # rubocop:disable PerceivedComplexity
       dir = @options[:pid_dir]
-      Dir.mkdir(dir) unless File.exist?(dir)
+      FileUtils.mkdir_p(dir) unless File.exist?(dir)
 
       if worker_pools
         setup_pools
       elsif @options[:identifier]
+        # rubocop:disable GuardClause
         if worker_count > 1
           raise ArgumentError, 'Cannot specify both --number-of-workers and --identifier'
         else
           run_process("delayed_job.#{@options[:identifier]}", @options)
         end
+        # rubocop:enable GuardClause
       else
         worker_count.times do |worker_index|
           process_name = worker_count == 1 ? 'delayed_job' : "delayed_job.#{worker_index}"
@@ -114,18 +127,19 @@ module Delayed
     end
 
     def run(worker_name = nil, options = {})
-      Dir.chdir(Rails.root)
+      Dir.chdir(root)
 
       Delayed::Worker.after_fork
-      Delayed::Worker.logger ||= Logger.new(File.join(Rails.root, 'log', 'delayed_job.log'))
+      Delayed::Worker.logger ||= Logger.new(File.join(@options[:log_dir], 'delayed_job.log'))
 
       worker = Delayed::Worker.new(options)
       worker.name_prefix = "#{worker_name} "
       worker.start
     rescue => e
-      Rails.logger.fatal e
       STDERR.puts e.message
-      exit 1
+      STDERR.puts e.backtrace
+      ::Rails.logger.fatal(e) if rails_logger_defined?
+      exit_with_error_status
     end
 
   private
@@ -134,13 +148,25 @@ module Delayed
       @worker_pools ||= []
 
       queues, worker_count = pool.split(':')
-      if ['*', '', nil].include?(queues)
-        queues = []
-      else
-        queues = queues.split(',')
-      end
+      queues = ['*', '', nil].include?(queues) ? [] : queues.split(',')
       worker_count = (worker_count || 1).to_i rescue 1
       @worker_pools << [queues, worker_count]
     end
+
+    def root
+      @root ||= rails_root_defined? ? ::Rails.root : DIR_PWD
+    end
+
+    def rails_root_defined?
+      defined?(::Rails.root)
+    end
+
+    def rails_logger_defined?
+      defined?(::Rails.logger)
+    end
+
+    def exit_with_error_status
+      exit 1
+    end
   end
 end

data/lib/delayed/exceptions.rb

@@ -8,5 +8,5 @@ module Delayed
     end
   end
 
-  class FatalBackendError < Exception; end
+  class FatalBackendError < RuntimeError; end
 end

data/lib/delayed/lifecycle.rb

@@ -1,5 +1,5 @@
 module Delayed
-  class InvalidCallback < Exception; end
+  class InvalidCallback < RuntimeError; end
 
   class Lifecycle
     EVENTS = {
@@ -10,7 +10,7 @@ module Delayed
       :error      => [:worker, :job],
       :failure    => [:worker, :job],
       :invoke_job => [:job]
-    }
+    }.freeze
 
     def initialize
       @callbacks = EVENTS.keys.each_with_object({}) do |e, hash|

data/lib/delayed/message_sending.rb

@@ -1,5 +1,3 @@
-require 'active_support/core_ext/module/aliasing'
-
 module Delayed
   class DelayProxy < Delayed::Compatibility.proxy_object_class
     def initialize(payload_class, target, options)
@@ -8,9 +6,11 @@ module Delayed
       @options = options
     end
 
+    # rubocop:disable MethodMissing
     def method_missing(method, *args)
       Job.enqueue({:payload_object => @payload_class.new(@target, method.to_sym, args)}.merge(@options))
     end
+    # rubocop:enable MethodMissing
   end
 
   module MessageSending
@@ -28,24 +28,36 @@ module Delayed
       warn '[DEPRECATION] `object.send_at(time, :method)` is deprecated. Use `object.delay(:run_at => time).method'
       __delay__(:run_at => time).__send__(method, *args)
     end
+  end
 
-    module ClassMethods
-      def handle_asynchronously(method, opts = {})
-        aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1 # rubocop:disable PerlBackrefs
-        with_method, without_method = "#{aliased_method}_with_delay#{punctuation}", "#{aliased_method}_without_delay#{punctuation}"
-        define_method(with_method) do |*args|
-          curr_opts = opts.clone
-          curr_opts.each_key do |key|
-            next unless (val = curr_opts[key]).is_a?(Proc)
-            curr_opts[key] = if val.arity == 1
-              val.call(self)
-            else
-              val.call
-            end
+  module MessageSendingClassMethods
+    def handle_asynchronously(method, opts = {}) # rubocop:disable PerceivedComplexity
+      aliased_method = method.to_s.sub(/([?!=])$/, '')
+      punctuation = $1 # rubocop:disable PerlBackrefs
+      with_method = "#{aliased_method}_with_delay#{punctuation}"
+      without_method = "#{aliased_method}_without_delay#{punctuation}"
+      define_method(with_method) do |*args|
+        curr_opts = opts.clone
+        curr_opts.each_key do |key|
+          next unless (val = curr_opts[key]).is_a?(Proc)
+          curr_opts[key] = if val.arity == 1
+            val.call(self)
+          else
+            val.call
           end
-          delay(curr_opts).__send__(without_method, *args)
         end
-        alias_method_chain method, :delay
+        delay(curr_opts).__send__(without_method, *args)
+      end
+
+      alias_method without_method, method
+      alias_method method, with_method
+
+      if public_method_defined?(without_method)
+        public method
+      elsif protected_method_defined?(without_method)
+        protected method
+      elsif private_method_defined?(without_method)
+        private method
       end
     end
   end

data/lib/delayed/performable_method.rb

@@ -1,11 +1,7 @@
-require 'active_support/core_ext/module/delegation'
-
 module Delayed
   class PerformableMethod
     attr_accessor :object, :method_name, :args
 
-    delegate :method, :to => :object
-
     def initialize(object, method_name, args)
       raise NoMethodError, "undefined method `#{method_name}' for #{object.inspect}" unless object.respond_to?(method_name, true)
 
@@ -30,9 +26,15 @@ module Delayed
       object.send(method_name, *args) if object
     end
 
+    def method(sym)
+      object.method(sym)
+    end
+
+    # rubocop:disable MethodMissing
     def method_missing(symbol, *args)
       object.send(symbol, *args)
     end
+    # rubocop:enable MethodMissing
 
     def respond_to?(symbol, include_private = false)
       super || object.respond_to?(symbol, include_private)

data/lib/delayed/psych_ext.rb

@@ -28,23 +28,29 @@ module Delayed
       end
 
       def visit_Psych_Nodes_Mapping(object) # rubocop:disable CyclomaticComplexity, MethodName, PerceivedComplexity
-        return revive(Psych.load_tags[object.tag], object) if Psych.load_tags[object.tag]
+        klass = Psych.load_tags[object.tag]
+        if klass
+          # Implementation changed here https://github.com/ruby/psych/commit/2c644e184192975b261a81f486a04defa3172b3f
+          # load_tags used to have class values, now the values are strings
+          klass = resolve_class(klass) if klass.is_a?(String)
+          return revive(klass, object)
+        end
 
         case object.tag
-        when /^!ruby\/object/
+        when %r{^!ruby/object}
           result = super
-          if defined?(ActiveRecord::Base) && result.is_a?(ActiveRecord::Base)
+          if jruby_is_seriously_borked && result.is_a?(ActiveRecord::Base)
             klass = result.class
             id = result[klass.primary_key]
             begin
-              klass.find(id)
+              klass.unscoped.find(id)
             rescue ActiveRecord::RecordNotFound => error # rubocop:disable BlockNesting
               raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
             end
           else
             result
           end
-        when /^!ruby\/ActiveRecord:(.+)$/
+        when %r{^!ruby/ActiveRecord:(.+)$}
           klass = resolve_class(Regexp.last_match[1])
           payload = Hash[*object.children.map { |c| accept c }]
           id = payload['attributes'][klass.primary_key]
@@ -54,7 +60,7 @@ module Delayed
           rescue ActiveRecord::RecordNotFound => error
             raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
           end
-        when /^!ruby\/Mongoid:(.+)$/
+        when %r{^!ruby/Mongoid:(.+)$}
           klass = resolve_class(Regexp.last_match[1])
           payload = Hash[*object.children.map { |c| accept c }]
           id = payload['attributes']['_id']
@@ -63,7 +69,7 @@ module Delayed
           rescue Mongoid::Errors::DocumentNotFound => error
             raise Delayed::DeserializationError, "Mongoid::Errors::DocumentNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
           end
-        when /^!ruby\/DataMapper:(.+)$/
+        when %r{^!ruby/DataMapper:(.+)$}
           klass = resolve_class(Regexp.last_match[1])
           payload = Hash[*object.children.map { |c| accept c }]
           begin
@@ -78,6 +84,13 @@ module Delayed
         end
       end
 
+      # defined? is triggering something really messed up in
+      # jruby causing both the if AND else clauses to execute,
+      # however if the check is run here, everything is fine
+      def jruby_is_seriously_borked
+        defined?(ActiveRecord::Base)
+      end
+
       def resolve_class(klass_name)
         return nil if !klass_name || klass_name.empty?
         klass_name.constantize

data/lib/delayed/railtie.rb

@@ -4,10 +4,6 @@ require 'rails'
 module Delayed
   class Railtie < Rails::Railtie
     initializer :after_initialize do
-      ActiveSupport.on_load(:action_mailer) do
-        ActionMailer::Base.extend(Delayed::DelayMail)
-      end
-
       Delayed::Worker.logger ||= if defined?(Rails)
         Rails.logger
       elsif defined?(RAILS_DEFAULT_LOGGER)

data/lib/delayed/recipes.rb

@@ -38,17 +38,17 @@ Capistrano::Configuration.instance.load do
 
     desc 'Stop the delayed_job process'
     task :stop, :roles => lambda { roles } do
-      run "cd #{current_path};#{rails_env} #{delayed_job_command} stop #{args}"
+      run "cd #{current_path} && #{rails_env} #{delayed_job_command} stop #{args}"
     end
 
     desc 'Start the delayed_job process'
     task :start, :roles => lambda { roles } do
-      run "cd #{current_path};#{rails_env} #{delayed_job_command} start #{args}"
+      run "cd #{current_path} && #{rails_env} #{delayed_job_command} start #{args}"
     end
 
     desc 'Restart the delayed_job process'
     task :restart, :roles => lambda { roles } do
-      run "cd #{current_path};#{rails_env} #{delayed_job_command} restart #{args}"
+      run "cd #{current_path} && #{rails_env} #{delayed_job_command} restart #{args}"
     end
   end
 end

data/lib/delayed/serialization/active_record.rb

@@ -1,7 +1,7 @@
 if defined?(ActiveRecord)
   module ActiveRecord
     class Base
-      yaml_as 'tag:ruby.yaml.org,2002:ActiveRecord'
+      yaml_tag 'tag:ruby.yaml.org,2002:ActiveRecord'
 
       def self.yaml_new(klass, _tag, val)
         klass.unscoped.find(val['attributes'][klass.primary_key])

data/lib/delayed/syck_ext.rb

@@ -1,5 +1,5 @@
 class Module
-  yaml_as 'tag:ruby.yaml.org,2002:module'
+  yaml_tag 'tag:ruby.yaml.org,2002:module'
 
   def self.yaml_new(_klass, _tag, val)
     val.constantize
@@ -20,7 +20,7 @@ class Module
 end
 
 class Class
-  yaml_as 'tag:ruby.yaml.org,2002:class'
+  yaml_tag 'tag:ruby.yaml.org,2002:class'
   remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
 end
 
@@ -29,7 +29,7 @@ class Struct
     # Constantize the object so that ActiveSupport can attempt
     # its auto loading magic. Will raise LoadError if not successful.
     name.constantize
-    "Struct::#{ name }"
+    "Struct::#{name}"
   end
 end
 

data/lib/delayed/tasks.rb

@@ -19,7 +19,7 @@ namespace :jobs do
       :min_priority => ENV['MIN_PRIORITY'],
       :max_priority => ENV['MAX_PRIORITY'],
       :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','),
-      :quiet => false
+      :quiet => ENV['QUIET']
     }
 
     @worker_options[:sleep_delay] = ENV['SLEEP_DELAY'].to_i if ENV['SLEEP_DELAY']

data/lib/delayed/worker.rb

@@ -1,20 +1,22 @@
 require 'timeout'
+require 'active_support/dependencies'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/class/attribute_accessors'
-require 'active_support/core_ext/kernel'
-require 'active_support/core_ext/enumerable'
+require 'active_support/hash_with_indifferent_access'
+require 'active_support/core_ext/hash/indifferent_access'
 require 'logger'
 require 'benchmark'
 
 module Delayed
   class Worker # rubocop:disable ClassLength
-    DEFAULT_LOG_LEVEL        = 'info'
+    DEFAULT_LOG_LEVEL        = 'info'.freeze
     DEFAULT_SLEEP_DELAY      = 5
     DEFAULT_MAX_ATTEMPTS     = 25
     DEFAULT_MAX_RUN_TIME     = 4.hours
     DEFAULT_DEFAULT_PRIORITY = 0
     DEFAULT_DELAY_JOBS       = true
-    DEFAULT_QUEUES           = []
+    DEFAULT_QUEUES           = [].freeze
+    DEFAULT_QUEUE_ATTRIBUTES = HashWithIndifferentAccess.new.freeze
     DEFAULT_READ_AHEAD       = 5
 
     cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time,
@@ -25,7 +27,7 @@ module Delayed
     # Named queue into which jobs are enqueued by default
     cattr_accessor :default_queue_name
 
-    cattr_reader :backend
+    cattr_reader :backend, :queue_attributes
 
     # name_prefix is ignored if name is set directly
     attr_accessor :name_prefix
@@ -38,11 +40,11 @@ module Delayed
       self.default_priority  = DEFAULT_DEFAULT_PRIORITY
       self.delay_jobs        = DEFAULT_DELAY_JOBS
       self.queues            = DEFAULT_QUEUES
+      self.queue_attributes  = DEFAULT_QUEUE_ATTRIBUTES
       self.read_ahead        = DEFAULT_READ_AHEAD
+      @lifecycle             = nil
     end
 
-    reset
-
     # Add or remove plugins in this list before the worker is instantiated
     self.plugins = [Delayed::Plugins::ClearLocks]
 
@@ -69,6 +71,11 @@ module Delayed
       silence_warnings { ::Delayed.const_set(:Job, backend) }
     end
 
+    # rubocop:disable ClassVars
+    def self.queue_attributes=(val)
+      @@queue_attributes = val.with_indifferent_access
+    end
+
     def self.guess_backend
       warn '[DEPRECATION] guess_backend is deprecated. Please remove it from your code.'
     end
@@ -97,13 +104,29 @@ module Delayed
     end
 
     def self.lifecycle
-      @lifecycle ||= Delayed::Lifecycle.new
+      # In case a worker has not been set up, job enqueueing needs a lifecycle.
+      setup_lifecycle unless @lifecycle
+
+      @lifecycle
+    end
+
+    def self.setup_lifecycle
+      @lifecycle = Delayed::Lifecycle.new
+      plugins.each { |klass| klass.new }
     end
 
     def self.reload_app?
       defined?(ActionDispatch::Reloader) && Rails.application.config.cache_classes == false
     end
 
+    def self.delay_job?(job)
+      if delay_jobs.is_a?(Proc)
+        delay_jobs.arity == 1 ? delay_jobs.call(job) : delay_jobs.call
+      else
+        delay_jobs
+      end
+    end
+
     def initialize(options = {})
       @quiet = options.key?(:quiet) ? options[:quiet] : true
       @failed_reserve_count = 0
@@ -112,7 +135,9 @@ module Delayed
         self.class.send("#{option}=", options[option]) if options.key?(option)
       end
 
-      plugins.each { |klass| klass.new }
+      # Reset lifecycle on the offhand chance that something lazily
+      # triggered its creation before all plugins had been registered.
+      self.class.setup_lifecycle
     end
 
     # Every worker has a unique name which by default is the pid of the process. There are some
@@ -121,7 +146,7 @@ module Delayed
     # it crashed before.
     def name
       return @name unless @name.nil?
-      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}" # rubocop:disable RescueModifier
+      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}"
     end
 
     # Sets the name of the worker.
@@ -151,7 +176,7 @@ module Delayed
             end
           end
 
-          count = @result.sum
+          count = @result[0] + @result[1]
 
           if count.zero?
             if self.class.exit_on_complete
@@ -181,7 +206,8 @@ module Delayed
     # Do num jobs and return stats on success/failure.
     # Exit early if interrupted.
     def work_off(num = 100)
-      success, failure = 0, 0
+      success = 0
+      failure = 0
 
       num.times do
         case reserve_and_run_one_job
@@ -190,7 +216,7 @@ module Delayed
         when false
           failure += 1
         else
-          break  # leave if no work could be done
+          break # leave if no work could be done
         end
         break if stop? # leave if we're exiting
       end
@@ -200,18 +226,20 @@ module Delayed
 
     def run(job)
       job_say job, 'RUNNING'
-      runtime =  Benchmark.realtime do
+      runtime = Benchmark.realtime do
         Timeout.timeout(max_run_time(job).to_i, WorkerTimeout) { job.invoke_job }
         job.destroy
       end
       job_say job, format('COMPLETED after %.4f', runtime)
-      return true  # did work
+      return true # did work
     rescue DeserializationError => error
-      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      job_say job, "FAILED permanently with #{error.class.name}: #{error.message}", 'error'
+
+      job.error = error
       failed(job)
-    rescue => error
+    rescue Exception => error # rubocop:disable RescueException
       self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, error) }
-      return false  # work failed
+      return false # work failed
     end
 
     # Reschedule the job in the future (when a job fails).
@@ -223,7 +251,7 @@ module Delayed
         job.unlock
         job.save!
       else
-        job_say job, "REMOVED permanently because of #{job.attempts} consecutive failures", 'error'
+        job_say job, "FAILED permanently because of #{job.attempts} consecutive failures", 'error'
         failed(job)
       end
     end
@@ -236,13 +264,13 @@ module Delayed
           say "Error when running failure callback: #{error}", 'error'
           say error.backtrace.join("\n"), 'error'
         ensure
-          self.class.destroy_failed_jobs ? job.destroy : job.fail!
+          job.destroy_failed_jobs? ? job.destroy : job.fail!
         end
       end
     end
 
     def job_say(job, text, level = default_log_level)
-      text = "Job #{job.name} (id=#{job.id}) #{text}"
+      text = "Job #{job.name} (id=#{job.id})#{say_queue(job.queue)} #{text}"
       say text, level
     end
 
@@ -267,8 +295,12 @@ module Delayed
 
   protected
 
+    def say_queue(queue)
+      " (queue=#{queue})" if queue
+    end
+
     def handle_failed_job(job, error)
-      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      job.error = error
       job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error'
       reschedule(job)
     end
@@ -294,8 +326,14 @@ module Delayed
 
     def reload!
       return unless self.class.reload_app?
-      ActionDispatch::Reloader.cleanup!
-      ActionDispatch::Reloader.prepare!
+      if defined?(ActiveSupport::Reloader)
+        Rails.application.reloader.reload!
+      else
+        ActionDispatch::Reloader.cleanup!
+        ActionDispatch::Reloader.prepare!
+      end
     end
   end
 end
+
+Delayed::Worker.reset