delayed_job_tgmerritt 4.0.7

data/lib/delayed/psych_ext.rb

@@ -0,0 +1,89 @@
+module Delayed
+  class PerformableMethod
+    # serialize to YAML
+    def encode_with(coder)
+      coder.map = {
+        'object' => object,
+        'method_name' => method_name,
+        'args' => args
+      }
+    end
+  end
+end
+
+module Psych
+  def self.load_dj(yaml)
+    result = parse(yaml)
+    result ? Delayed::PsychExt::ToRuby.create.accept(result) : result
+  end
+end
+
+module Delayed
+  module PsychExt
+    class ToRuby < Psych::Visitors::ToRuby
+      unless respond_to?(:create)
+        def self.create
+          new
+        end
+      end
+
+      def visit_Psych_Nodes_Mapping(object) # rubocop:disable CyclomaticComplexity, MethodName, PerceivedComplexity
+        return revive(Psych.load_tags[object.tag], object) if Psych.load_tags[object.tag]
+
+        case object.tag
+        when /^!ruby\/object/
+          result = super
+          if defined?(ActiveRecord::Base) && result.is_a?(ActiveRecord::Base)
+            klass = result.class
+            id = result[klass.primary_key]
+            begin
+              klass.find(id)
+            rescue ActiveRecord::RecordNotFound => error # rubocop:disable BlockNesting
+              raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
+            end
+          else
+            result
+          end
+        when /^!ruby\/ActiveRecord:(.+)$/
+          klass = resolve_class(Regexp.last_match[1])
+          payload = Hash[*object.children.map { |c| accept c }]
+          id = payload['attributes'][klass.primary_key]
+          id = id.value if defined?(ActiveRecord::Attribute) && id.is_a?(ActiveRecord::Attribute)
+          begin
+            klass.unscoped.find(id)
+          rescue ActiveRecord::RecordNotFound => error
+            raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
+          end
+        when /^!ruby\/Mongoid:(.+)$/
+          klass = resolve_class(Regexp.last_match[1])
+          payload = Hash[*object.children.map { |c| accept c }]
+          id = payload['attributes']['_id']
+          begin
+            klass.find(id)
+          rescue Mongoid::Errors::DocumentNotFound => error
+            raise Delayed::DeserializationError, "Mongoid::Errors::DocumentNotFound, class: #{klass}, primary key: #{id} (#{error.message})"
+          end
+        when /^!ruby\/DataMapper:(.+)$/
+          klass = resolve_class(Regexp.last_match[1])
+          payload = Hash[*object.children.map { |c| accept c }]
+          begin
+            primary_keys = klass.properties.select(&:key?)
+            key_names = primary_keys.map { |p| p.name.to_s }
+            klass.get!(*key_names.map { |k| payload['attributes'][k] })
+          rescue DataMapper::ObjectNotFoundError => error
+            raise Delayed::DeserializationError, "DataMapper::ObjectNotFoundError, class: #{klass} (#{error.message})"
+          end
+        else
+          super
+        end
+      end
+
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        klass_name.constantize
+      rescue
+        super
+      end
+    end
+  end
+end

data/lib/delayed/railtie.rb

@@ -0,0 +1,22 @@
+require 'delayed_job'
+require 'rails'
+
+module Delayed
+  class Railtie < Rails::Railtie
+    initializer :after_initialize do
+      ActiveSupport.on_load(:action_mailer) do
+        ActionMailer::Base.extend(Delayed::DelayMail)
+      end
+
+      Delayed::Worker.logger ||= if defined?(Rails)
+        Rails.logger
+      elsif defined?(RAILS_DEFAULT_LOGGER)
+        RAILS_DEFAULT_LOGGER
+      end
+    end
+
+    rake_tasks do
+      load 'delayed/tasks.rb'
+    end
+  end
+end

data/lib/delayed/recipes.rb

@@ -0,0 +1,54 @@
+# Capistrano Recipes for managing delayed_job
+#
+# Add these callbacks to have the delayed_job process restart when the server
+# is restarted:
+#
+#   after "deploy:stop",    "delayed_job:stop"
+#   after "deploy:start",   "delayed_job:start"
+#   after "deploy:restart", "delayed_job:restart"
+#
+# If you want to use command line options, for example to start multiple workers,
+# define a Capistrano variable delayed_job_args:
+#
+#   set :delayed_job_args, "-n 2"
+#
+# If you've got delayed_job workers running on a servers, you can also specify
+# which servers have delayed_job running and should be restarted after deploy.
+#
+#   set :delayed_job_server_role, :worker
+#
+
+Capistrano::Configuration.instance.load do
+  namespace :delayed_job do
+    def rails_env
+      fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
+    end
+
+    def args
+      fetch(:delayed_job_args, '')
+    end
+
+    def roles
+      fetch(:delayed_job_server_role, :app)
+    end
+
+    def delayed_job_command
+      fetch(:delayed_job_command, 'script/delayed_job')
+    end
+
+    desc 'Stop the delayed_job process'
+    task :stop, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{delayed_job_command} stop #{args}"
+    end
+
+    desc 'Start the delayed_job process'
+    task :start, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{delayed_job_command} start #{args}"
+    end
+
+    desc 'Restart the delayed_job process'
+    task :restart, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{delayed_job_command} restart #{args}"
+    end
+  end
+end

data/lib/delayed/serialization/active_record.rb

@@ -0,0 +1,17 @@
+if defined?(ActiveRecord)
+  module ActiveRecord
+    class Base
+      yaml_as 'tag:ruby.yaml.org,2002:ActiveRecord'
+
+      def self.yaml_new(klass, _tag, val)
+        klass.unscoped.find(val['attributes'][klass.primary_key])
+      rescue ActiveRecord::RecordNotFound
+        raise Delayed::DeserializationError, "ActiveRecord::RecordNotFound, class: #{klass} , primary key: #{val['attributes'][klass.primary_key]}"
+      end
+
+      def to_yaml_properties
+        ['@attributes']
+      end
+    end
+  end
+end

data/lib/delayed/syck_ext.rb

@@ -0,0 +1,42 @@
+class Module
+  yaml_as 'tag:ruby.yaml.org,2002:module'
+
+  def self.yaml_new(_klass, _tag, val)
+    val.constantize
+  end
+
+  def to_yaml(options = {})
+    YAML.quick_emit(nil, options) do |out|
+      out.scalar(taguri, name, :plain)
+    end
+  end
+
+  def yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    name
+  end
+end
+
+class Class
+  yaml_as 'tag:ruby.yaml.org,2002:class'
+  remove_method :to_yaml if respond_to?(:to_yaml) && method(:to_yaml).owner == Class # use Module's to_yaml
+end
+
+class Struct
+  def self.yaml_tag_read_class(name)
+    # Constantize the object so that ActiveSupport can attempt
+    # its auto loading magic. Will raise LoadError if not successful.
+    name.constantize
+    "Struct::#{ name }"
+  end
+end
+
+module YAML
+  def load_dj(yaml)
+    # See https://github.com/dtao/safe_yaml
+    # When the method is there, we need to load our YAML like this...
+    respond_to?(:unsafe_load) ? load(yaml, :safe => false) : load(yaml)
+  end
+end

data/lib/delayed/tasks.rb

@@ -0,0 +1,39 @@
+namespace :jobs do
+  desc 'Clear the delayed_job queue.'
+  task :clear => :environment do
+    Delayed::Job.delete_all
+  end
+
+  desc 'Start a delayed_job worker.'
+  task :work => :environment_options do
+    Delayed::Worker.new(@worker_options).start
+  end
+
+  desc 'Start a delayed_job worker and exit when all available jobs are complete.'
+  task :workoff => :environment_options do
+    Delayed::Worker.new(@worker_options.merge(:exit_on_complete => true)).start
+  end
+
+  task :environment_options => :environment do
+    @worker_options = {
+      :min_priority => ENV['MIN_PRIORITY'],
+      :max_priority => ENV['MAX_PRIORITY'],
+      :queues => (ENV['QUEUES'] || ENV['QUEUE'] || '').split(','),
+      :quiet => false
+    }
+
+    @worker_options[:sleep_delay] = ENV['SLEEP_DELAY'].to_i if ENV['SLEEP_DELAY']
+    @worker_options[:read_ahead] = ENV['READ_AHEAD'].to_i if ENV['READ_AHEAD']
+  end
+
+  desc "Exit with error status if any jobs older than max_age seconds haven't been attempted yet."
+  task :check, [:max_age] => :environment do |_, args|
+    args.with_defaults(:max_age => 300)
+
+    unprocessed_jobs = Delayed::Job.where('attempts = 0 AND created_at < ?', Time.now - args[:max_age].to_i).count
+
+    if unprocessed_jobs > 0
+      raise "#{unprocessed_jobs} jobs older than #{args[:max_age]} seconds have not been processed yet"
+    end
+  end
+end

data/lib/delayed/worker.rb

@@ -0,0 +1,312 @@
+require 'timeout'
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/class/attribute_accessors'
+require 'active_support/core_ext/kernel'
+require 'active_support/core_ext/enumerable'
+require 'logger'
+require 'benchmark'
+
+module Delayed
+  class Worker # rubocop:disable ClassLength
+    DEFAULT_LOG_LEVEL        = 'info'
+    DEFAULT_SLEEP_DELAY      = 5
+    DEFAULT_MAX_ATTEMPTS     = 25
+    DEFAULT_MAX_RUN_TIME     = 4.hours
+    DEFAULT_DEFAULT_PRIORITY = 0
+    DEFAULT_DELAY_JOBS       = true
+    DEFAULT_QUEUES           = []
+    DEFAULT_READ_AHEAD       = 5
+
+    cattr_accessor :min_priority, :max_priority, :max_attempts, :max_run_time,
+                   :default_priority, :sleep_delay, :logger, :delay_jobs, :queues,
+                   :read_ahead, :plugins, :destroy_failed_jobs, :exit_on_complete,
+                   :default_log_level
+
+    # Named queue into which jobs are enqueued by default
+    cattr_accessor :default_queue_name
+
+    cattr_reader :backend
+
+    # name_prefix is ignored if name is set directly
+    attr_accessor :name_prefix
+
+    def self.reset
+      self.default_log_level = DEFAULT_LOG_LEVEL
+      self.sleep_delay       = DEFAULT_SLEEP_DELAY
+      self.max_attempts      = DEFAULT_MAX_ATTEMPTS
+      self.max_run_time      = DEFAULT_MAX_RUN_TIME
+      self.default_priority  = DEFAULT_DEFAULT_PRIORITY
+      self.delay_jobs        = DEFAULT_DELAY_JOBS
+      self.queues            = DEFAULT_QUEUES
+      self.read_ahead        = DEFAULT_READ_AHEAD
+      @lifecycle             = nil
+    end
+
+    reset
+
+    # Add or remove plugins in this list before the worker is instantiated
+    self.plugins = [Delayed::Plugins::ClearLocks]
+
+    # By default failed jobs are destroyed after too many attempts. If you want to keep them around
+    # (perhaps to inspect the reason for the failure), set this to false.
+    self.destroy_failed_jobs = true
+
+    # By default, Signals INT and TERM set @exit, and the worker exits upon completion of the current job.
+    # If you would prefer to raise a SignalException and exit immediately you can use this.
+    # Be aware daemons uses TERM to stop and restart
+    # false - No exceptions will be raised
+    # :term - Will only raise an exception on TERM signals but INT will wait for the current job to finish
+    # true - Will raise an exception on TERM and INT
+    cattr_accessor :raise_signal_exceptions
+    self.raise_signal_exceptions = false
+
+    def self.backend=(backend)
+      if backend.is_a? Symbol
+        require "delayed/serialization/#{backend}"
+        require "delayed/backend/#{backend}"
+        backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
+      end
+      @@backend = backend # rubocop:disable ClassVars
+      silence_warnings { ::Delayed.const_set(:Job, backend) }
+    end
+
+    def self.guess_backend
+      warn '[DEPRECATION] guess_backend is deprecated. Please remove it from your code.'
+    end
+
+    def self.before_fork
+      unless @files_to_reopen
+        @files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+          @files_to_reopen << file unless file.closed?
+        end
+      end
+
+      backend.before_fork
+    end
+
+    def self.after_fork
+      # Re-open file handles
+      @files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          file.sync = true
+        rescue ::Exception # rubocop:disable HandleExceptions, RescueException
+        end
+      end
+      backend.after_fork
+    end
+
+    def self.lifecycle
+      # In case a worker has not been set up, job enqueueing needs a lifecycle.
+      setup_lifecycle unless @lifecycle
+
+      @lifecycle
+    end
+
+    def self.setup_lifecycle
+      @lifecycle = Delayed::Lifecycle.new
+      plugins.each { |klass| klass.new }
+    end
+
+    def self.reload_app?
+      defined?(ActionDispatch::Reloader) && Rails.application.config.cache_classes == false
+    end
+
+    def initialize(options = {})
+      @quiet = options.key?(:quiet) ? options[:quiet] : true
+      @failed_reserve_count = 0
+
+      [:min_priority, :max_priority, :sleep_delay, :read_ahead, :queues, :exit_on_complete].each do |option|
+        self.class.send("#{option}=", options[option]) if options.key?(option)
+      end
+
+      # Reset lifecycle on the offhand chance that something lazily
+      # triggered its creation before all plugins had been registered.
+      self.class.setup_lifecycle
+    end
+
+    # Every worker has a unique name which by default is the pid of the process. There are some
+    # advantages to overriding this with something which survives worker restarts:  Workers can
+    # safely resume working on tasks which are locked by themselves. The worker will assume that
+    # it crashed before.
+    def name
+      return @name unless @name.nil?
+      "#{@name_prefix}host:#{Socket.gethostname} pid:#{Process.pid}" rescue "#{@name_prefix}pid:#{Process.pid}" # rubocop:disable RescueModifier
+    end
+
+    # Sets the name of the worker.
+    # Setting the name to nil will reset the default worker name
+    attr_writer :name
+
+    def start # rubocop:disable CyclomaticComplexity, PerceivedComplexity
+      trap('TERM') do
+        Thread.new { say 'Exiting...' }
+        stop
+        raise SignalException, 'TERM' if self.class.raise_signal_exceptions
+      end
+
+      trap('INT') do
+        Thread.new { say 'Exiting...' }
+        stop
+        raise SignalException, 'INT' if self.class.raise_signal_exceptions && self.class.raise_signal_exceptions != :term
+      end
+
+      say 'Starting job worker'
+
+      self.class.lifecycle.run_callbacks(:execute, self) do
+        loop do
+          self.class.lifecycle.run_callbacks(:loop, self) do
+            @realtime = Benchmark.realtime do
+              @result = work_off
+            end
+          end
+
+          count = @result.sum
+
+          if count.zero?
+            if self.class.exit_on_complete
+              say 'No more jobs available. Exiting'
+              break
+            elsif !stop?
+              sleep(self.class.sleep_delay)
+              reload!
+            end
+          else
+            say format("#{count} jobs processed at %.4f j/s, %d failed", count / @realtime, @result.last)
+          end
+
+          break if stop?
+        end
+      end
+    end
+
+    def stop
+      @exit = true
+    end
+
+    def stop?
+      !!@exit
+    end
+
+    # Do num jobs and return stats on success/failure.
+    # Exit early if interrupted.
+    def work_off(num = 100)
+      success, failure = 0, 0
+
+      num.times do
+        case reserve_and_run_one_job
+        when true
+          success += 1
+        when false
+          failure += 1
+        else
+          break  # leave if no work could be done
+        end
+        break if stop? # leave if we're exiting
+      end
+
+      [success, failure]
+    end
+
+    def run(job)
+      job_say job, 'RUNNING'
+      runtime =  Benchmark.realtime do
+        Timeout.timeout(max_run_time(job).to_i, WorkerTimeout) { job.invoke_job }
+        job.destroy
+      end
+      job_say job, format('COMPLETED after %.4f', runtime)
+      return true  # did work
+    rescue DeserializationError => error
+      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      failed(job)
+    rescue => error
+      self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, error) }
+      return false  # work failed
+    end
+
+    # Reschedule the job in the future (when a job fails).
+    # Uses an exponential scale depending on the number of failed attempts.
+    def reschedule(job, time = nil)
+      if (job.attempts += 1) < max_attempts(job)
+        time ||= job.reschedule_at
+        job.run_at = time
+        job.unlock
+        job.save!
+      else
+        job_say job, "REMOVED permanently because of #{job.attempts} consecutive failures", 'error'
+        failed(job)
+      end
+    end
+
+    def failed(job)
+      self.class.lifecycle.run_callbacks(:failure, self, job) do
+        begin
+          job.hook(:failure)
+        rescue => error
+          say "Error when running failure callback: #{error}", 'error'
+          say error.backtrace.join("\n"), 'error'
+        ensure
+          self.class.destroy_failed_jobs ? job.destroy : job.fail!
+        end
+      end
+    end
+
+    def job_say(job, text, level = default_log_level)
+      text = "Job #{job.name} (id=#{job.id}) #{text}"
+      say text, level
+    end
+
+    def say(text, level = default_log_level)
+      text = "[Worker(#{name})] #{text}"
+      puts text unless @quiet
+      return unless logger
+      # TODO: Deprecate use of Fixnum log levels
+      unless level.is_a?(String)
+        level = Logger::Severity.constants.detect { |i| Logger::Severity.const_get(i) == level }.to_s.downcase
+      end
+      logger.send(level, "#{Time.now.strftime('%FT%T%z')}: #{text}")
+    end
+
+    def max_attempts(job)
+      job.max_attempts || self.class.max_attempts
+    end
+
+    def max_run_time(job)
+      job.max_run_time || self.class.max_run_time
+    end
+
+  protected
+
+    def handle_failed_job(job, error)
+      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", 'error'
+      reschedule(job)
+    end
+
+    # Run the next job we can get an exclusive lock on.
+    # If no jobs are left we return nil
+    def reserve_and_run_one_job
+      job = reserve_job
+      self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) } if job
+    end
+
+    def reserve_job
+      job = Delayed::Job.reserve(self)
+      @failed_reserve_count = 0
+      job
+    rescue ::Exception => error # rubocop:disable RescueException
+      say "Error while reserving job: #{error}"
+      Delayed::Job.recover_from(error)
+      @failed_reserve_count += 1
+      raise FatalBackendError if @failed_reserve_count >= 10
+      nil
+    end
+
+    def reload!
+      return unless self.class.reload_app?
+      ActionDispatch::Reloader.cleanup!
+      ActionDispatch::Reloader.prepare!
+    end
+  end
+end