delayed 0.1.0

data/Rakefile

@@ -0,0 +1,35 @@
+require 'bundler/gem_helper'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+
+ADAPTERS = %w(mysql2 postgresql sqlite3).freeze
+
+ADAPTERS.each do |adapter|
+  desc "Run RSpec code examples for #{adapter} adapter"
+  RSpec::Core::RakeTask.new(adapter => "#{adapter}:adapter")
+
+  namespace adapter do
+    task :adapter do
+      ENV['ADAPTER'] = adapter
+    end
+  end
+end
+
+task :adapter do
+  ENV['ADAPTER'] = nil
+end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+if ENV['APPRAISAL_INITIALIZED'] || ENV['CI']
+  tasks = ADAPTERS + [:adapter]
+  tasks += [:rubocop] unless ENV['CI']
+
+  task default: tasks
+else
+  require 'appraisal'
+  Appraisal::Task.new
+  task default: :appraisal
+end

data/lib/delayed.rb

@@ -0,0 +1,72 @@
+require 'active_support'
+require 'active_support/core_ext/numeric/time'
+require 'delayed/exceptions'
+require 'delayed/message_sending'
+require 'delayed/performable_method'
+require 'delayed/yaml_ext'
+require 'delayed/lifecycle'
+require 'delayed/runnable'
+require 'delayed/priority'
+require 'delayed/monitor'
+require 'delayed/plugin'
+require 'delayed/plugins/connection'
+require 'delayed/plugins/instrumentation'
+require 'delayed/backend/base'
+require 'delayed/backend/job_preparer'
+require 'delayed/worker'
+require 'delayed/railtie' if defined?(Rails::Railtie)
+
+ActiveSupport.on_load(:active_record) do
+  require 'delayed/serialization/active_record'
+  require 'delayed/job'
+end
+
+ActiveSupport.on_load(:active_job) do
+  require 'delayed/active_job_adapter'
+  ActiveJob::QueueAdapters::DelayedAdapter = Class.new(Delayed::ActiveJobAdapter)
+
+  include Delayed::ActiveJobAdapter::EnqueuingPatch
+end
+
+ActiveSupport.on_load(:action_mailer) do
+  require 'delayed/performable_mailer'
+  ActionMailer::Base.extend(Delayed::DelayMail)
+  ActionMailer::Parameterized::Mailer.include(Delayed::DelayMail) if defined?(ActionMailer::Parameterized::Mailer)
+end
+
+module Delayed
+  autoload :PerformableMailer, 'delayed/performable_mailer'
+
+  mattr_accessor(:default_log_level) { 'info'.freeze }
+  mattr_accessor(:plugins) do
+    [
+      Delayed::Plugins::Instrumentation,
+      Delayed::Plugins::Connection,
+    ]
+  end
+
+  def self.lifecycle
+    setup_lifecycle unless @lifecycle
+    @lifecycle
+  end
+
+  def self.setup_lifecycle
+    @lifecycle = Delayed::Lifecycle.new
+    plugins.each { |klass| klass.new }
+  end
+
+  def self.logger
+    @logger ||= Rails.logger
+  end
+
+  def self.logger=(value)
+    @logger = value
+  end
+
+  def self.say(message, level = default_log_level)
+    logger&.send(level, message)
+  end
+end
+
+Object.include Delayed::MessageSending
+Module.include Delayed::MessageSendingClassMethods

data/lib/delayed/active_job_adapter.rb

@@ -0,0 +1,65 @@
+module Delayed
+  class ActiveJobAdapter
+    def enqueue(job)
+      _enqueue(job)
+    end
+
+    def enqueue_at(job, timestamp)
+      _enqueue(job, run_at: Time.at(timestamp)) # rubocop:disable Rails/TimeZone
+    end
+
+    private
+
+    def _enqueue(job, opts = {})
+      opts.merge!({ queue: job.queue_name, priority: job.priority }.compact)
+        .merge!(job.provider_attributes || {})
+
+      Delayed::Job.enqueue(JobWrapper.new(job.serialize), opts).tap do |dj|
+        job.provider_job_id = dj.id
+      end
+    end
+
+    module EnqueuingPatch
+      def self.included(klass)
+        klass.prepend PrependedMethods
+        klass.attr_accessor :provider_attributes
+      end
+
+      module PrependedMethods
+        def enqueue(opts = {})
+          raise "`:run_at` is not supported. Use `:wait_until` instead." if opts.key?(:run_at)
+
+          self.provider_attributes = opts.except(:wait, :wait_until, :queue, :priority)
+          opts[:priority] = Delayed::Priority.new(opts[:priority]) if opts.key?(:priority)
+          super(opts)
+        end
+      end
+    end
+  end
+
+  class JobWrapper # rubocop:disable Betterment/ActiveJobPerformable
+    attr_accessor :job_data
+
+    delegate_missing_to :job
+
+    def initialize(job_data)
+      @job_data = job_data
+    end
+
+    def display_name
+      job_data['job_class']
+    end
+
+    def perform
+      ActiveJob::Callbacks.run_callbacks(:execute) do
+        job.perform_now
+      end
+    end
+
+    private
+
+    def job
+      @job ||= ActiveJob::Base.deserialize(job_data) if job_data
+    end
+  end
+end

data/lib/delayed/backend/base.rb

@@ -0,0 +1,166 @@
+module Delayed
+  module Backend
+    module Base
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+        # Add a job to the queue
+        def enqueue(*args)
+          job_options = Delayed::Backend::JobPreparer.new(*args).prepare
+          enqueue_job(job_options)
+        end
+
+        def enqueue_job(options)
+          new(options).tap do |job|
+            Delayed.lifecycle.run_callbacks(:enqueue, job) do
+              job.hook(:enqueue)
+              Delayed::Worker.delay_job?(job) ? job.save : job.invoke_job
+            end
+          end
+        end
+
+        def reserve(worker, max_run_time = Worker.max_run_time)
+          # We get up to 5 jobs from the db. In case we cannot get exclusive access to a job we try the next.
+          # this leads to a more even distribution of jobs across the worker processes
+          claims = 0
+          find_available(worker.name, worker.read_ahead, max_run_time).select do |job|
+            next if claims >= worker.max_claims
+
+            job.lock_exclusively!(max_run_time, worker.name).tap do |result|
+              claims += 1 if result
+            end
+          end
+        end
+
+        # Allow the backend to attempt recovery from reserve errors
+        def recover_from(_error); end
+
+        def work_off(num = 100)
+          warn '[DEPRECATION] `Delayed::Job.work_off` is deprecated. Use `Delayed::Worker.new.work_off instead.'
+          Delayed::Worker.new.work_off(num)
+        end
+      end
+
+      attr_reader :error
+
+      def error=(error)
+        @error = error
+        self.last_error = "#{error.message}\n#{error.backtrace.join("\n")}" if respond_to?(:last_error=)
+      end
+
+      def failed?
+        !!failed_at
+      end
+      alias failed failed?
+
+      ParseObjectFromYaml = %r{!ruby/\w+:([^\s]+)}.freeze # rubocop:disable Naming/ConstantName
+
+      def name # rubocop:disable Metrics/AbcSize
+        @name ||= payload_object.job_data['job_class'] if payload_object.respond_to?(:job_data)
+        @name ||= payload_object.display_name if payload_object.respond_to?(:display_name)
+        @name ||= payload_object.class.name
+      rescue DeserializationError
+        ParseObjectFromYaml.match(handler)[1]
+      end
+
+      def priority
+        Priority.new(super)
+      end
+
+      def priority=(value)
+        super(Priority.new(value))
+      end
+
+      def payload_object=(object)
+        @payload_object = object
+        self.handler = YAML.dump_dj(object)
+      end
+
+      def payload_object
+        @payload_object ||= YAML.load_dj(handler)
+      rescue TypeError, LoadError, NameError, ArgumentError, SyntaxError, Psych::SyntaxError => e
+        raise DeserializationError, "Job failed to load: #{e.message}. Handler: #{handler.inspect}"
+      end
+
+      def invoke_job
+        Delayed::Worker.lifecycle.run_callbacks(:invoke_job, self) do
+          hook :before
+          payload_object.perform
+          hook :success
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          hook :error, e
+          raise e
+        ensure
+          hook :after
+        end
+      end
+
+      # Unlock this job (note: not saved to DB)
+      def unlock
+        self.locked_at    = nil
+        self.locked_by    = nil
+      end
+
+      def hook(name, *args)
+        if payload_object.respond_to?(name)
+          if payload_object.is_a?(Delayed::JobWrapper)
+            return if name == :enqueue # this callback is not supported due to method naming conflicts.
+
+            warn '[DEPRECATION] Job hook methods (`before`, `after`, `success`, etc) are deprecated. Use ActiveJob callbacks instead.'
+          end
+
+          method = payload_object.method(name)
+          method.arity.zero? ? method.call : method.call(self, *args)
+        end
+      rescue DeserializationError
+      end
+
+      def reschedule_at
+        if payload_object.respond_to?(:reschedule_at)
+          payload_object.reschedule_at(self.class.db_time_now, attempts)
+        else
+          self.class.db_time_now + (attempts**4) + 5
+        end
+      end
+
+      def max_attempts
+        payload_object.max_attempts if payload_object.respond_to?(:max_attempts)
+      end
+
+      def max_run_time
+        return unless payload_object.respond_to?(:max_run_time)
+        return unless (run_time = payload_object.max_run_time)
+
+        if run_time > Delayed::Worker.max_run_time
+          Delayed::Worker.max_run_time
+        else
+          run_time
+        end
+      end
+
+      def destroy_failed_jobs?
+        payload_object.respond_to?(:destroy_failed_jobs?) ? payload_object.destroy_failed_jobs? : Delayed::Worker.destroy_failed_jobs
+      rescue DeserializationError
+        Delayed::Worker.destroy_failed_jobs
+      end
+
+      def fail!
+        self.failed_at = self.class.db_time_now
+        save!
+      end
+
+      protected
+
+      def set_default_run_at
+        self.run_at ||= self.class.db_time_now
+      end
+
+      # Call during reload operation to clear out internal state
+      def reset
+        @payload_object = nil
+      end
+    end
+  end
+end

data/lib/delayed/backend/job_preparer.rb

@@ -0,0 +1,43 @@
+module Delayed
+  module Backend
+    class JobPreparer
+      attr_reader :options, :args
+
+      def initialize(*args)
+        @options = args.extract_options!.dup
+        @args = args
+      end
+
+      def prepare
+        set_payload
+        set_queue_name
+        set_priority
+        handle_deprecation
+        options
+      end
+
+      private
+
+      def set_payload
+        options[:payload_object] ||= args.shift
+      end
+
+      def set_queue_name
+        options[:queue] ||= options[:payload_object].queue_name if options[:payload_object].respond_to?(:queue_name)
+        options[:queue] ||= Delayed::Worker.default_queue_name
+      end
+
+      def set_priority
+        options[:priority] ||= options[:payload_object].priority if options[:payload_object].respond_to?(:priority)
+        options[:priority] ||= Delayed::Worker.default_priority
+      end
+
+      def handle_deprecation
+        unless options[:payload_object].respond_to?(:perform)
+          raise ArgumentError,
+                'Cannot enqueue items which do not respond to perform'
+        end
+      end
+    end
+  end
+end

data/lib/delayed/exceptions.rb

@@ -0,0 +1,14 @@
+require 'timeout'
+
+module Delayed
+  class WorkerTimeout < RuntimeError
+    def message
+      seconds = Delayed::Worker.max_run_time.to_i
+      "#{super} (Delayed::Worker.max_run_time is only #{seconds} second#{seconds == 1 ? '' : 's'})"
+    end
+  end
+
+  class FatalBackendError < RuntimeError; end
+
+  class DeserializationError < StandardError; end
+end

data/lib/delayed/job.rb

@@ -0,0 +1,250 @@
+module Delayed
+  class Job < ::ActiveRecord::Base
+    include Delayed::Backend::Base
+
+    scope :by_priority, lambda { order("priority ASC, run_at ASC") }
+    scope :min_priority, lambda { |priority| where("priority >= ?", priority) if priority }
+    scope :max_priority, lambda { |priority| where("priority <= ?", priority) if priority }
+    scope :for_queues, lambda { |queues| where(queue: queues) if queues.any? }
+
+    scope :locked, -> { where.not(locked_at: nil) }
+    scope :erroring, -> { where.not(last_error: nil) }
+    scope :failed, -> { where.not(failed_at: nil) }
+    scope :not_locked, -> { where(locked_at: nil) }
+    scope :not_failed, -> { where(failed_at: nil) }
+    scope :workable, ->(timestamp) { not_locked.not_failed.where("run_at <= ?", timestamp) }
+    scope :working, -> { locked.not_failed }
+
+    before_save :set_default_run_at
+
+    REENQUEUE_BUFFER = 30.seconds
+
+    def self.set_delayed_job_table_name
+      delayed_job_table_name = "#{::ActiveRecord::Base.table_name_prefix}delayed_jobs"
+      self.table_name = delayed_job_table_name
+    end
+
+    set_delayed_job_table_name
+
+    def self.ready_to_run(worker_name, max_run_time)
+      where(
+        "((run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) OR locked_by = ?) AND failed_at IS NULL",
+        db_time_now,
+        db_time_now - (max_run_time + REENQUEUE_BUFFER),
+        worker_name,
+      )
+    end
+
+    # When a worker is exiting, make sure we don't have any locked jobs.
+    def self.clear_locks!(worker_name)
+      where(locked_by: worker_name).update_all(locked_by: nil, locked_at: nil)
+    end
+
+    def self.reserve(worker, max_run_time = Worker.max_run_time)
+      ready_scope =
+        ready_to_run(worker.name, max_run_time)
+          .min_priority(worker.min_priority)
+          .max_priority(worker.max_priority)
+          .for_queues(worker.queues)
+          .by_priority
+
+      ActiveSupport::Notifications.instrument('delayed.worker.reserve_jobs', worker_tags(worker)) do
+        reserve_with_scope(ready_scope, worker, db_time_now)
+      end
+    end
+
+    def self.reserve_with_scope(ready_scope, worker, now)
+      case connection.adapter_name
+        when "PostgreSQL", "PostGIS"
+          reserve_with_scope_using_optimized_postgres(ready_scope, worker, now)
+        when "MySQL", "Mysql2"
+          reserve_with_scope_using_optimized_mysql(ready_scope, worker, now)
+        when "MSSQL", "Teradata"
+          reserve_with_scope_using_optimized_mssql(ready_scope, worker, now)
+        # Fallback for unknown / other DBMS
+        else
+          reserve_with_scope_using_default_sql(ready_scope, worker, now)
+      end
+    end
+
+    def self.reserve_with_scope_using_default_sql(ready_scope, worker, now)
+      # This is our old fashion, tried and true, but possibly slower lookup
+      # Instead of reading the entire job record for our detect loop, we select only the id,
+      # and only read the full job record after we've successfully locked the job.
+      # This can have a noticable impact on large read_ahead configurations and large payload jobs.
+      attrs = { locked_at: now, locked_by: worker.name }
+
+      jobs = []
+      ready_scope.limit(worker.read_ahead).select(:id).each do |job|
+        break if jobs.count >= worker.max_claims
+        next unless ready_scope.where(id: job.id).update_all(attrs) == 1
+
+        jobs << job.reload
+      end
+
+      jobs
+    end
+
+    def self.reserve_with_scope_using_optimized_postgres(ready_scope, worker, now) # rubocop:disable Metrics/AbcSize
+      # Custom SQL required for PostgreSQL because postgres does not support UPDATE...LIMIT
+      # This locks the single record 'FOR UPDATE' in the subquery
+      # http://www.postgresql.org/docs/9.0/static/sql-select.html#SQL-FOR-UPDATE-SHARE
+      # Note: active_record would attempt to generate UPDATE...LIMIT like
+      # SQL for Postgres if we use a .limit() filter, but it would not
+      # use 'FOR UPDATE' and we would have many locking conflicts
+      table = connection.quote_table_name(table_name)
+
+      # Rather than relying on a primary key, we use "WHERE ctid =", resulting in a fast 'Tid Scan'.
+      if worker.max_claims > 1
+        subquery = ready_scope.limit(worker.max_claims).lock("FOR UPDATE SKIP LOCKED").select("ctid").to_sql
+        sql = "UPDATE #{table} SET locked_at = ?, locked_by = ? WHERE ctid = ANY (ARRAY (#{subquery})) RETURNING *"
+      else
+        subquery = ready_scope.limit(1).lock("FOR UPDATE SKIP LOCKED").select("ctid").to_sql
+        sql = "UPDATE #{table} SET locked_at = ?, locked_by = ? WHERE ctid = (#{subquery}) RETURNING *"
+      end
+
+      find_by_sql([sql, now, worker.name]).sort_by(&:priority)
+    end
+
+    def self.reserve_with_scope_using_optimized_mysql(ready_scope, worker, now)
+      # Removing the millisecond precision from now(time object)
+      # MySQL 5.6.4 onwards millisecond precision exists, but the
+      # datetime object created doesn't have precision, so discarded
+      # while updating. But during the where clause, for mysql(>=5.6.4),
+      # it queries with precision as well. So removing the precision
+      now = now.change(usec: 0)
+      # Despite MySQL's support of UPDATE...LIMIT, it has an optimizer bug
+      # that results in filesorts rather than index scans, which is very
+      # expensive with a large number of jobs in the table:
+      # http://bugs.mysql.com/bug.php?id=74049
+      # The PostgreSQL and MSSQL reserve strategies, while valid syntax in
+      # MySQL, result in deadlocks so we use a SELECT then UPDATE strategy
+      # that is more likely to false-negative when attempting to reserve
+      # jobs in parallel but doesn't rely on subselects or transactions.
+
+      # Also, we are fetching multiple candidate_jobs at a time to try to
+      # avoid the situation where multiple workers try to grab the same
+      # job at the same time. That previously had caused poor performance
+      # since ready_scope.where(id: job.id) would return nothing even
+      # though there was a large number of jobs on the queue.
+      attrs = { locked_at: now, locked_by: worker.name }
+
+      jobs = []
+      ready_scope.limit(worker.read_ahead).each do |job|
+        break if jobs.count >= worker.max_claims
+        next unless ready_scope.where(id: job.id).update_all(attrs) == 1
+
+        job.assign_attributes(attrs)
+        job.send(:changes_applied)
+        jobs << job
+      end
+
+      jobs
+    end
+
+    def self.reserve_with_scope_using_optimized_mssql(ready_scope, worker, now)
+      # The MSSQL driver doesn't generate a limit clause when update_all
+      # is called directly
+      subsubquery_sql = ready_scope.limit(1).to_sql
+      # select("id") doesn't generate a subquery, so force a subquery
+      subquery_sql = "SELECT id FROM (#{subsubquery_sql}) AS x"
+      quoted_table_name = connection.quote_table_name(table_name)
+      sql = "UPDATE #{quoted_table_name} SET locked_at = ?, locked_by = ? WHERE id IN (#{subquery_sql})"
+      count = connection.execute(sanitize_sql([sql, now, worker.name]))
+      return [] if count.zero?
+
+      # MSSQL JDBC doesn't support OUTPUT INSERTED.* for returning a result set, so query locked row
+      where(locked_at: now, locked_by: worker.name, failed_at: nil)
+    end
+
+    # Get the current time (GMT or local depending on DB)
+    # Note: This does not ping the DB to get the time, so all your clients
+    # must have syncronized clocks.
+    def self.db_time_now
+      if Time.zone
+        Time.zone.now
+      elsif ::ActiveRecord::Base.default_timezone == :utc
+        Time.now.utc
+      else
+        Time.current
+      end
+    end
+
+    def self.worker_tags(worker)
+      {
+        min_priority: worker.min_priority,
+        max_priority: worker.max_priority,
+        max_claims: worker.max_claims,
+        read_ahead: worker.read_ahead,
+        queues: worker.queues,
+        table: table_name,
+        database: database_name,
+        database_adapter: database_adapter_name,
+        worker: worker,
+      }
+    end
+
+    def self.database_name
+      connection_config[:database]
+    end
+
+    def self.database_adapter_name
+      connection_config[:adapter]
+    end
+
+    if ActiveRecord.gem_version >= Gem::Version.new('6.1')
+      def self.connection_config
+        connection_db_config.configuration_hash
+      end
+    end
+
+    def reload(*args)
+      reset
+      super
+    end
+
+    def alert_age
+      if payload_object.respond_to?(:alert_age)
+        payload_object.alert_age
+      else
+        priority.alert_age
+      end
+    end
+
+    def alert_run_time
+      if payload_object.respond_to?(:alert_run_time)
+        payload_object.alert_run_time
+      else
+        priority.alert_run_time
+      end
+    end
+
+    def alert_attempts
+      if payload_object.respond_to?(:alert_attempts)
+        payload_object.alert_attempts
+      else
+        priority.alert_attempts
+      end
+    end
+
+    def age
+      [(locked_at || self.class.db_time_now) - run_at, 0].max
+    end
+
+    def run_time
+      self.class.db_time_now - locked_at if locked_at
+    end
+
+    def age_alert?
+      alert_age&.<= age
+    end
+
+    def run_time_alert?
+      alert_run_time&.<= run_time if run_time # locked_at may be `nil` if `delay_jobs` is false
+    end
+
+    def attempts_alert?
+      alert_attempts&.<= attempts
+    end
+  end
+end