delayed 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f978a0dbfd75659bb230acd0941387a03ad410861aeb4f50f9f13e1e0bca68c
-  data.tar.gz: c8505cfdc0730e2ce2ed8f21f92ec25e9fcd332fead3591650db9b6e95090205
+  metadata.gz: a42f2f7d06ec85c051b3a72e7e8644a750e703e6994cd89e75b151e098810fde
+  data.tar.gz: 122a0e93f5790543eca0484c27f52eff1472f693d1c8eee5c0aef27e1cace575
 SHA512:
-  metadata.gz: e6c4e6b4f82a6bb7687130a7c069b5a3a3334902762d7b106ac36f049f2cdeca047aa260cf97cbbfa7ea3791cfd44ea8611eb274a217dddf8c3d8e7d8ac2ed22
-  data.tar.gz: ed3f2531eb4baba2029d080696267c8c578f1d573fdc919e63edefcd2151be577e64a246babe14bfe500154a11d0abdbbdb045921ae95f2f0102dddb18894e32
+  metadata.gz: 2238ccb8c25ef5e483faf3a6bfa43f1c72a2b81e2cd5312071d8dbc72ef96397e3b6dceaac61db962028524e9a6b06769005ce64e9a5a1b7b818a81a40554c95
+  data.tar.gz: b255dee5f83ffe62155edaf65e552a3f6042656b60515b7721a0707ff78c2ed8f284d5e298b9448c2ee34cb8d574447619f9e78ec92da5f1dee461f4ebbe4b70

data/README.md

@@ -72,8 +72,10 @@ gem 'delayed'
 
 Then run `bundle install`.
 
-Before you can enqueue and run jobs, you will need a jobs table. You can create this table by
-running the following command:
+#### Database Setup
+
+Before you can enqueue and run jobs, you will need a `delayed_jobs` table. You
+can create this table by running the following command:
 
 ```bash
 rake delayed:install:migrations
@@ -81,11 +83,21 @@ rails db:migrate
 ```
 
 This will produce a series of migrations ready to be run in sequence.
-(Re-running the command should be safe and will **not** duplicate previous
-migrations, but if you have an existing `delayed_jobs` table, you may need to
-adjust the generated migrations to avoid conflicts.)
+(Re-running the command will **not** duplicate previous migrations, so you
+should aim to re-run it after each new release of the gem to pick up any new
+schema changes.)
+
+⚠️ **Important: If you already have an existing `delayed_jobs` table...** ⚠️
+- ...you may need to adjust the generated migrations to avoid conflicts.
+- ...you should inspect the migrations closely, as they are not guaranteed to be
+  "safe" to run with zero downtime. (We recommend using the
+  [strong_migrations](https://github.com/ankane/strong_migrations) gem to help
+  identify any unsafe operations, and at a certain scale all bets are off
+  without careful testing!)
+
+#### ActiveJob Setup
 
-Then, to use this background job processor with ActiveJob, add the following to your application config:
+To use this background job processor with ActiveJob, add the following to your application config:
 
 ```ruby
 config.active_job.queue_adapter = :delayed
@@ -591,4 +603,5 @@ creating a new issue to get early feedback on your proposed change.
 * Fork the project and create a new branch for your contribution.
 * Write your contribution (and any applicable test coverage).
 * Make sure all tests pass (`bundle exec rake`).
+    * If you are changing SQL queries, re-record snapshots with `RECORD_SNAPSHOTS=1`
 * Submit a pull request.

data/Rakefile

@@ -3,7 +3,7 @@ Bundler::GemHelper.install_tasks
 
 require 'rspec/core/rake_task'
 
-ADAPTERS = %w(mysql2 postgresql sqlite3).freeze
+ADAPTERS = %w(postgresql sqlite3 mysql2).freeze
 
 ADAPTERS.each do |adapter|
   desc "Run RSpec code examples for #{adapter} adapter"
@@ -20,6 +20,10 @@ task :adapter do
   ENV['ADAPTER'] = nil
 end
 
+if ENV['RECORD_SNAPSHOTS']
+  `rm -rf spec/**/__snapshots__`
+end
+
 require 'rubocop/rake_task'
 RuboCop::RakeTask.new
 

data/app/models/delayed/job.rb

@@ -2,18 +2,43 @@ module Delayed
   class Job < ::ActiveRecord::Base
     include Delayed::Backend::Base
 
-    scope :by_priority, lambda { order("priority ASC, run_at ASC") }
-    scope :min_priority, lambda { |priority| where("priority >= ?", priority) if priority }
-    scope :max_priority, lambda { |priority| where("priority <= ?", priority) if priority }
-    scope :for_queues, lambda { |queues| where(queue: queues) if queues.any? }
-
-    scope :locked, -> { where.not(locked_at: nil) }
-    scope :erroring, -> { where.not(last_error: nil) }
+    # worker config scopes
+    scope :by_priority, -> { order(:priority, :run_at) }
+    scope :min_priority, ->(priority) { where(arel_table[:priority].gteq(priority)) if priority }
+    scope :max_priority, ->(priority) { where(arel_table[:priority].lteq(priority)) if priority }
+    scope :for_queues, ->(queues) { where(queue: queues) if queues.any? }
+
+    # high-level queue states (live => erroring => failed)
+    scope :live, -> { where(failed_at: nil) }
+    scope :erroring, -> { where(arel_table[:attempts].gt(0)).merge(unscoped.live) }
     scope :failed, -> { where.not(failed_at: nil) }
-    scope :not_locked, -> { where(locked_at: nil) }
-    scope :not_failed, -> { where(failed_at: nil) }
-    scope :workable, ->(timestamp) { not_locked.not_failed.where("run_at <= ?", timestamp) }
-    scope :working, -> { locked.not_failed }
+
+    # live queue states (future vs pending)
+    scope :future, ->(as_of = db_time_now) { merge(unscoped.live).where(arel_table[:run_at].gt(as_of)) }
+    scope :pending, ->(as_of = db_time_now) { merge(unscoped.live).where(arel_table[:run_at].lteq(as_of)) }
+
+    # pending queue states (claimed vs claimable)
+    scope :claimed, ->(as_of = db_time_now) {
+      where(arel_table[:locked_at].gteq(db_time_now - lock_timeout))
+        .merge(unscoped.pending(as_of))
+    }
+    scope :claimed_by, ->(worker, as_of = db_time_now) {
+      where(locked_by: worker.name)
+        .claimed(as_of)
+    }
+    scope :claimable, ->(as_of = db_time_now) {
+      where(locked_at: nil)
+        .or(where(arel_table[:locked_at].lt(db_time_now - lock_timeout)))
+        .merge(unscoped.pending(as_of))
+    }
+    scope :claimable_by, ->(worker, as_of = db_time_now) {
+      claimable(as_of)
+        .or(claimed_by(worker, as_of))
+        .min_priority(worker.min_priority)
+        .max_priority(worker.max_priority)
+        .for_queues(worker.queues)
+        .by_priority
+    }
 
     before_save :set_default_run_at, :set_name
 
@@ -26,30 +51,18 @@ module Delayed
 
     set_delayed_job_table_name
 
-    def self.ready_to_run(worker_name, max_run_time)
-      where(
-        "((run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) OR locked_by = ?) AND failed_at IS NULL",
-        db_time_now,
-        db_time_now - (max_run_time + REENQUEUE_BUFFER),
-        worker_name,
-      )
+    def self.lock_timeout
+      Worker.max_run_time + REENQUEUE_BUFFER
     end
 
     # When a worker is exiting, make sure we don't have any locked jobs.
-    def self.clear_locks!(worker_name)
-      where(locked_by: worker_name).update_all(locked_by: nil, locked_at: nil)
+    def self.clear_locks!(worker)
+      claimed_by(worker).update_all(locked_by: nil, locked_at: nil)
     end
 
-    def self.reserve(worker, max_run_time = Worker.max_run_time)
-      ready_scope =
-        ready_to_run(worker.name, max_run_time)
-          .min_priority(worker.min_priority)
-          .max_priority(worker.max_priority)
-          .for_queues(worker.queues)
-          .by_priority
-
+    def self.reserve(worker, as_of = db_time_now)
       ActiveSupport::Notifications.instrument('delayed.worker.reserve_jobs', worker_tags(worker)) do
-        reserve_with_scope(ready_scope, worker, db_time_now)
+        reserve_with_scope(claimable_by(worker, as_of), worker, as_of)
       end
     end
 

data/db/migrate/1_create_delayed_jobs.rb

@@ -12,8 +12,6 @@ class CreateDelayedJobs < ActiveRecord::Migration[6.0]
       table.string :queue                              # The name of the queue this job is in
       table.timestamps null: true
     end
-
-    add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority"
   end
 
   def self.down

data/db/migrate/3_add_index_to_delayed_jobs_name.rb

@@ -1,11 +1,19 @@
 class AddIndexToDelayedJobsName < ActiveRecord::Migration[6.0]
-  disable_ddl_transaction!
+  include Delayed::Helpers::Migration
+
+  # Set to the maximum amount of time you want this migration to run:
+  WAIT_TIMEOUT = 5.minutes
+
+  # Concurrent index creation cannot be run inside a transaction:
+  disable_ddl_transaction! if concurrent_index_creation_supported?
 
   def change
-    if connection.adapter_name == 'PostgreSQL'
-      add_index :delayed_jobs, :name, algorithm: :concurrently
-    else
-      add_index :delayed_jobs, :name
-    end
+    opts = {}
+
+    # Postgres supports creating indexes concurrently, which avoids locking the table
+    # while the index is building:
+    opts[:algorithm] = :concurrently if concurrent_index_creation_supported?
+
+    upsert_index :delayed_jobs, :name, wait_timeout: WAIT_TIMEOUT, **opts
   end
 end

data/db/migrate/4_index_live_jobs.rb

@@ -0,0 +1,33 @@
+class IndexLiveJobs < ActiveRecord::Migration[6.0]
+  include Delayed::Helpers::Migration
+
+  # Set to the maximum amount of time you want this migration to run:
+  WAIT_TIMEOUT = 5.minutes
+
+  # Concurrent index creation cannot be run inside a transaction:
+  disable_ddl_transaction! if concurrent_index_creation_supported?
+
+  def change
+    opts = { name: 'idx_delayed_jobs_live' }
+    columns = %i(priority run_at locked_at queue attempts)
+
+    # Postgres supports creating indexes concurrently,
+    # which avoids locking the table while the index is building:
+    opts[:algorithm] = :concurrently if concurrent_index_creation_supported?
+
+    if connection.supports_partial_index?
+      # Postgres and SQLite both support partial indexes, allowing us to pre-filter out failed jobs:
+      opts[:where] = '(failed_at IS NULL)'
+    else
+      # If partial indexes aren't supported, failed_at will be included in the primary index:
+      columns = %i(failed_at) + columns
+    end
+
+    # On PostgreSQL, we do not index `locked_at` or `locked_by` to optimize for HOT updates during pickup.
+    # See: https://www.postgresql.org/docs/current/storage-hot.html
+    # (On other databases, we can include `locked_at` to allow for more "covering" index lookups)
+    columns -= %i(locked_at) if connection.adapter_name == 'PostgreSQL'
+
+    upsert_index :delayed_jobs, columns, wait_timeout: WAIT_TIMEOUT, **opts
+  end
+end

data/db/migrate/5_index_failed_jobs.rb

@@ -0,0 +1,24 @@
+class IndexFailedJobs < ActiveRecord::Migration[6.0]
+  include Delayed::Helpers::Migration
+
+  # Set to the maximum amount of time you want this migration to run:
+  WAIT_TIMEOUT = 5.minutes
+
+  # Concurrent index creation cannot be run inside a transaction:
+  disable_ddl_transaction! if concurrent_index_creation_supported?
+
+  def change
+    # You can delete this migration if your database does not support partial indexes.
+    return unless connection.supports_partial_index?
+
+    # If partial indexes are supported, then the "live" index does not cover failed jobs.
+    # To aid in monitoring, this adds a separate (smaller) index for failed jobs:
+    opts = { name: 'idx_delayed_jobs_failed', where: '(failed_at IS NOT NULL)' }
+
+    # Postgres supports creating indexes concurrently, which avoids locking the table
+    # while the index is building:
+    opts[:algorithm] = :concurrently if concurrent_index_creation_supported?
+
+    upsert_index :delayed_jobs, %i(priority queue), wait_timeout: WAIT_TIMEOUT, **opts
+  end
+end

data/db/migrate/6_set_postgres_fillfactor.rb

@@ -0,0 +1,31 @@
+class SetPostgresFillfactor < ActiveRecord::Migration[6.0]
+  # On PostgreSQL, we do not index `locked_at` or `locked_by` to optimize for "HOT updates" during pickup.
+  # See: https://www.postgresql.org/docs/current/storage-hot.html
+  #
+  # To increase the odds that a given page has room for a HOT update, we reduce the
+  # "fillfactor" (percentage filled by default), and set a more aggressive autovacume target:
+  def up
+    return unless connection.adapter_name == 'PostgreSQL'
+
+    execute <<~SQL
+      ALTER TABLE delayed_jobs SET (
+        autovacuum_vacuum_scale_factor = 0.02,
+        fillfactor = 33
+      );
+      ALTER INDEX idx_delayed_jobs_live SET (
+        fillfactor = 33
+      );
+    SQL
+  end
+
+  def down
+    return unless connection.adapter_name == 'PostgreSQL'
+
+    execute <<~SQL
+      ALTER TABLE delayed_jobs
+        RESET (autovacuum_vacuum_scale_factor, fillfactor);
+      ALTER INDEX idx_delayed_jobs_live
+        RESET (fillfactor);
+    SQL
+  end
+end

data/db/migrate/7_remove_legacy_index.rb

@@ -0,0 +1,12 @@
+class RemoveLegacyIndex < ActiveRecord::Migration[6.0]
+  include Delayed::Helpers::Migration
+
+  disable_ddl_transaction! if concurrent_index_creation_supported?
+
+  def change
+    opts = { name: 'delayed_jobs_priority' }
+    opts[:algorithm] = :concurrently if concurrent_index_creation_supported?
+
+    remove_index_if_exists :delayed_jobs, %i(priority run_at), **opts, wait_timeout: 5.minutes
+  end
+end

data/lib/delayed/backend/job_preparer.rb

@@ -12,6 +12,7 @@ module Delayed
         set_payload
         set_queue_name
         set_priority
+        handle_dst
         handle_deprecation
         options
       end
@@ -32,6 +33,24 @@ module Delayed
         options[:priority] ||= Delayed::Worker.default_priority
       end
 
+      def scheduled_into_fall_back_hour?
+        options[:run_at] &&
+          !options[:run_at].in_time_zone.dst? &&
+          (options[:run_at] - 1.hour).dst?
+      end
+
+      def handle_dst
+        # The DB column does not retain timezone information. As a result, if we
+        # are running with `:local` timezone, then any future-scheduled jobs
+        # that fall into the "fall back" DST transition need to rounded up to
+        # the later hour or they will cause a "spinloop" of immediate retries.
+        if Job.default_timezone == :local && scheduled_into_fall_back_hour?
+          run_at_was = options[:run_at]
+          options[:run_at] = (run_at_was + 1.hour).beginning_of_hour
+          Delayed.say("Adjusted run_at from #{run_at_was} to #{options[:run_at]} to account for fall back DST transition", :warn)
+        end
+      end
+
       def handle_deprecation
         unless options[:payload_object].respond_to?(:perform)
           raise ArgumentError,

data/lib/delayed/exceptions.rb

@@ -1,7 +1,10 @@
 require 'timeout'
 
 module Delayed
-  class WorkerTimeout < RuntimeError
+  # We inherit from Exception because we want timeouts to bubble up to the
+  # worker thread where they can be handled appropriately. (If we inherited from
+  # StandardError, jobs are more likely to inadvertently `rescue` it directly.)
+  class WorkerTimeout < Exception # rubocop:disable Lint/InheritException
     def message
       seconds = Delayed::Worker.max_run_time.to_i
       "#{super} (Delayed::Worker.max_run_time is only #{seconds} second#{seconds == 1 ? '' : 's'})"

data/lib/delayed/helpers/migration.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Helpers
+    module Migration
+      def self.included(base)
+        base.extend(ClassMethods)
+        delegate :concurrent_index_creation_supported?, to: :class
+      end
+
+      module ClassMethods
+        def concurrent_index_creation_supported?
+          connection.index_algorithms.key?(:concurrently)
+        end
+      end
+
+      def upsert_index(*args, **opts)
+        dir(:up) { _add_or_replace_index(*args, **opts) }
+        dir(:down) { _drop_index_if_exists(*args, **opts) }
+      end
+
+      def remove_index_if_exists(*args, **opts)
+        dir(:up) { _drop_index_if_exists(*args, **opts) }
+        dir(:down) { _add_or_replace_index(*args, **opts) }
+      end
+
+      RETRY_EXCEPTIONS = [
+        ActiveRecord::LockWaitTimeout,
+        ActiveRecord::StatementTimeout,
+        (PG::LockNotAvailable if defined?(PG::LockNotAvailable)),
+      ].compact.freeze
+
+      def with_retry_loop(wait_timeout: 5.minutes, **opts)
+        with_timeouts(**opts) do
+          loop do
+            yield
+            break
+          rescue *RETRY_EXCEPTIONS => e
+            raise if Delayed::Job.db_time_now - @migration_start > wait_timeout
+
+            Delayed.logger.warn("Index creation failed for #{opts[:name]}: #{e.message}. Retrying...")
+          end
+        end
+      end
+
+      def with_timeouts(statement_timeout: 1.minute, lock_timeout: 5.seconds)
+        dir(:both) { set_timeouts!(statement_timeout: statement_timeout, lock_timeout: lock_timeout) }
+        yield
+      ensure
+        dir(:both) { set_timeouts!(statement_timeout: nil, lock_timeout: nil) }
+      end
+
+      private
+
+      def _add_or_replace_index(table, columns, **opts)
+        index = _lookup_index(table, columns, **opts)
+        if index && !_index_matches?(index, **opts)
+          Delayed.logger.warn("Recreating index #{index.name} (is invalid or does not match desired options)")
+          _drop_index(table, name: index.name, **opts)
+        end
+        _add_index(table, columns, **opts) if !index || !_index_matches?(index, **opts)
+      end
+
+      def _drop_index_if_exists(table, columns, **opts)
+        index = _lookup_index(table, columns, **opts)
+        _drop_index(table, name: index.name, **opts) if index
+      end
+
+      def _add_index(*args, **opts)
+        index_opts = opts.slice!(:wait_timeout, :statement_timeout, :lock_timeout)
+        with_retry_loop(**opts) { add_index(*args, **index_opts) }
+      end
+
+      def _drop_index(table, name:, **opts)
+        opts.slice!(:wait_timeout, :statement_timeout, :lock_timeout)
+        with_retry_loop(**opts) { remove_index(table, name: name) }
+      end
+
+      def _lookup_index(table, columns, **opts)
+        connection.indexes(table).find { |idx| idx.name == opts[:name] || idx.columns == Array(columns).map(&:to_s) }
+      end
+
+      def _index_matches?(index, **opts)
+        using_default = :btree unless connection.adapter_name == 'SQLite'
+
+        { unique: false, where: nil, using: using_default, include: nil, valid?: true }.all? do |key, default|
+          !index.respond_to?(key) || opts.fetch(key, default) == index.public_send(key)
+        end
+      end
+
+      def dir(direction, &block)
+        reversible do |dir|
+          dir.up(&block) if %i(up both).include?(direction)
+          dir.down(&block) if %i(down both).include?(direction)
+        end
+      end
+
+      def set_timeouts!(statement_timeout:, lock_timeout:)
+        case connection.adapter_name
+        when 'PostgreSQL'
+          execute("SET statement_timeout TO #{pg_seconds(statement_timeout) || 'DEFAULT'};")
+          execute("SET lock_timeout TO #{pg_seconds(lock_timeout) || 'DEFAULT'};")
+        when "MySQL", "Mysql2"
+          execute("SET SESSION wait_timeout = #{statement_timeout&.seconds || 'DEFAULT'};")
+          execute("SET SESSION lock_wait_timeout = #{lock_timeout&.seconds || 'DEFAULT'};")
+        else
+          Delayed.logger.info("[delayed] #{connection.adapter_name} does not support setting statement or lock timeouts (skipping).")
+        end
+      end
+
+      def pg_seconds(duration)
+        "'#{duration.seconds}s'" if duration
+      end
+    end
+  end
+end

data/lib/delayed/monitor.rb

@@ -20,21 +20,27 @@ module Delayed
     def initialize
       @jobs = Job.group(priority_case_statement).group(:queue)
       @jobs = @jobs.where(queue: Worker.queues) if Worker.queues.any?
+      @memo = {}
     end
 
     def run!
+      @memo = {}
       ActiveSupport::Notifications.instrument('delayed.monitor.run', default_tags) do
         METRICS.each { |metric| emit_metric!(metric) }
       end
       interruptable_sleep(sleep_delay)
     end
 
+    def query_for(metric)
+      send(:"#{metric}_grouped")
+    end
+
     private
 
     attr_reader :jobs
 
     def emit_metric!(metric)
-      send(:"#{metric}_grouped").reverse_merge(default_results).each do |(priority, queue), value|
+      query_for(metric).reverse_merge(default_results).each do |(priority, queue), value|
         ActiveSupport::Notifications.instrument(
           "delayed.job.#{metric}",
           default_tags.merge(priority: Priority.new(priority).to_s, queue: queue, value: value),
@@ -63,15 +69,19 @@ module Delayed
     end
 
     def count_grouped
-      jobs.count
+      if Job.connection.supports_partial_index?
+        failed_count_grouped.merge(jobs.live.count) { |_, l, f| l + f }
+      else
+        jobs.count
+      end
     end
 
     def future_count_grouped
-      jobs.where("run_at > ?", Job.db_time_now).count
+      jobs.future.count
     end
 
     def locked_count_grouped
-      jobs.locked.count
+      @memo[:locked_count_grouped] ||= jobs.claimed.count
     end
 
     def erroring_count_grouped
@@ -79,7 +89,7 @@ module Delayed
     end
 
     def failed_count_grouped
-      jobs.failed.count
+      @memo[:failed_count_grouped] ||= jobs.failed.count
     end
 
     def max_lock_age_grouped
@@ -102,19 +112,19 @@ module Delayed
     end
 
     def workable_count_grouped
-      jobs.workable(Job.db_time_now).count
+      jobs.claimable.count
     end
 
-    def working_count_grouped
-      jobs.working.count
-    end
+    alias working_count_grouped locked_count_grouped
 
     def oldest_locked_job_grouped
-      jobs.working.select("#{priority_case_statement} AS priority, queue, MIN(locked_at) AS locked_at")
+      jobs.claimed
+        .select("#{priority_case_statement} AS priority, queue, MIN(locked_at) AS locked_at")
     end
 
     def oldest_workable_job_grouped
-      jobs.workable(Job.db_time_now).select("(#{priority_case_statement}) AS priority, queue, MIN(run_at) AS run_at")
+      @memo[:oldest_workable_job_grouped] ||= jobs.claimable
+        .select("(#{priority_case_statement}) AS priority, queue, MIN(run_at) AS run_at")
     end
 
     def priority_case_statement

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = '1.2.1'
+  VERSION = '2.0.0'
 end

data/lib/delayed/worker.rb

@@ -83,7 +83,7 @@ module Delayed
     end
 
     def on_exit!
-      Delayed::Job.clear_locks!(name)
+      Delayed::Job.clear_locks!(self)
     end
 
     # Do num jobs and return stats on success/failure.

data/lib/delayed.rb

@@ -13,6 +13,7 @@ require 'delayed/plugins/connection'
 require 'delayed/plugins/instrumentation'
 require 'delayed/backend/base'
 require 'delayed/backend/job_preparer'
+require 'delayed/helpers/migration'
 require 'delayed/worker'
 require 'delayed/job_wrapper'