async-background 0.7.1 → 1.0.0

data/lib/async/background/queue/client.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
 
 require_relative '../clock'
+require_relative 'store'
 
 module Async
   module Background
     module Queue
-      EMPTY_OPTIONS = {}.freeze
-
       class Client
         include Clock
 
@@ -15,17 +14,17 @@ module Async
           @notifier = notifier
         end
 
-        def push(class_name, args = [], run_at = nil, options: {})
+        def push(class_name, args = EMPTY_ARGS, run_at = nil, options: EMPTY_OPTIONS)
           id = @store.enqueue(class_name, args, run_at, options: options)
           @notifier&.notify_all
           id
         end
 
-        def push_in(delay, class_name, args = [], options: {})
+        def push_in(delay, class_name, args = EMPTY_ARGS, options: EMPTY_OPTIONS)
           push(class_name, args, realtime_now + delay.to_f, options: options)
         end
 
-        def push_at(time, class_name, args = [], options: {})
+        def push_at(time, class_name, args = EMPTY_ARGS, options: EMPTY_OPTIONS)
           run_at = time.respond_to?(:to_f) ? time.to_f : time
           push(class_name, args, run_at, options: options)
         end
@@ -34,19 +33,27 @@ module Async
       class << self
         attr_accessor :default_client
 
-        def enqueue(job_class, *args, options: {})
+        def migrate!(path: Store.default_path, options: {})
+          Store.migrate!(path: path, options: options)
+        end
+
+        def prepare_dashboard!(path: Store.default_path, options: {})
+          Store.prepare_dashboard!(path: path, options: options)
+        end
+
+        def enqueue(job_class, *args, options: EMPTY_OPTIONS)
           ensure_configured!
-          default_client.push(resolve_class_name(job_class), args, nil, options: build_options(job_class, options))
+          default_client.push(resolve_class_name(job_class), normalized_args(args), nil, options: build_options(job_class, options))
         end
 
-        def enqueue_in(delay, job_class, *args, options: {})
+        def enqueue_in(delay, job_class, *args, options: EMPTY_OPTIONS)
           ensure_configured!
-          default_client.push_in(delay, resolve_class_name(job_class), args, options: build_options(job_class, options))
+          default_client.push_in(delay, resolve_class_name(job_class), normalized_args(args), options: build_options(job_class, options))
         end
 
-        def enqueue_at(time, job_class, *args, options: {})
+        def enqueue_at(time, job_class, *args, options: EMPTY_OPTIONS)
           ensure_configured!
-          default_client.push_at(time, resolve_class_name(job_class), args, options: build_options(job_class, options))
+          default_client.push_at(time, resolve_class_name(job_class), normalized_args(args), options: build_options(job_class, options))
         end
 
         private
@@ -55,8 +62,11 @@ module Async
         private_constant :RETRY_KEYS
 
         def build_options(job_class, call_site)
-          call_site ||= {}
-          merged = resolve_options(job_class).merge(call_site.compact)
+          call_site ||= EMPTY_OPTIONS
+          class_options = resolve_options(job_class)
+          return EMPTY_OPTIONS if class_options.empty? && call_site.empty?
+
+          merged = class_options.merge(call_site.compact)
           apply_retry_overrides!(merged, call_site)
 
           merged.empty? ? EMPTY_OPTIONS : Job::Options.new(**merged).to_h.compact
@@ -72,6 +82,8 @@ module Async
           raise "Async::Background::Queue not configured" unless default_client
         end
 
+        def normalized_args(args) = args.empty? ? EMPTY_ARGS : args
+
         def resolve_class_name(job_class)
           return job_class if job_class.is_a?(String)
           return job_class.name if job_class.respond_to?(:perform_now)
@@ -80,9 +92,15 @@ module Async
         end
 
         def resolve_options(job_class)
-          return {} unless job_class.respond_to?(:resolve_options)
+          return EMPTY_OPTIONS unless job_class.respond_to?(:resolve_options)
+
+          options = if job_class.respond_to?(:queue_options)
+            job_class.queue_options
+          else
+            job_class.resolve_options
+          end
 
-          job_class.resolve_options.dup
+          options.empty? ? EMPTY_OPTIONS : options
         end
       end
     end

data/lib/async/background/queue/options.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Queue
+      EMPTY_ARGS = [].freeze
+      EMPTY_OPTIONS = {}.freeze
+
+      SYNCHRONOUS_LEVELS = {normal: 'NORMAL', full: 'FULL', extra: 'EXTRA'}.freeze
+      WAL_AUTOCHECKPOINT_RANGE = 100..10_000
+      DEFAULT_STORE_OPTIONS = {mmap: true, synchronous: :normal, wal_autocheckpoint: 1_000}.freeze
+      DEFAULTS = DEFAULT_STORE_OPTIONS
+      MMAP_SIZE = 268_435_456
+
+      StoreOptions = Data.define(:mmap, :synchronous, :wal_autocheckpoint) do
+        def self.build(value = {})
+          return value if value.is_a?(self)
+
+          new(**DEFAULT_STORE_OPTIONS, **value)
+        end
+
+        def initialize(mmap:, synchronous:, wal_autocheckpoint:)
+          validate_mmap!(mmap)
+          validate_synchronous!(synchronous)
+          validate_wal_autocheckpoint!(wal_autocheckpoint)
+
+          super
+        end
+
+        def synchronous_pragma = SYNCHRONOUS_LEVELS.fetch(synchronous)
+        def mmap_size = mmap ? MMAP_SIZE : 0
+
+        def pragma_sql
+          <<~SQL
+            PRAGMA journal_mode       = WAL;
+            PRAGMA synchronous        = #{synchronous_pragma};
+            PRAGMA mmap_size          = #{mmap_size};
+            PRAGMA cache_size         = -16000;
+            PRAGMA temp_store         = MEMORY;
+            PRAGMA journal_size_limit = 67108864;
+            PRAGMA wal_autocheckpoint = #{wal_autocheckpoint};
+          SQL
+        end
+
+        private
+
+        def validate_mmap!(value)
+          return if value == true || value == false
+
+          raise ArgumentError, "mmap must be true or false, got #{value.inspect}"
+        end
+
+        def validate_synchronous!(value)
+          return if SYNCHRONOUS_LEVELS.key?(value)
+
+          raise ArgumentError,
+                "synchronous must be one of #{SYNCHRONOUS_LEVELS.keys.inspect}, got #{value.inspect}"
+        end
+
+        def validate_wal_autocheckpoint!(value)
+          return if value.is_a?(Integer) && WAL_AUTOCHECKPOINT_RANGE.cover?(value)
+
+          raise ArgumentError,
+                "wal_autocheckpoint must be an Integer in #{WAL_AUTOCHECKPOINT_RANGE}, " \
+                "got #{value.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/async/background/queue/schema.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require_relative 'sql'
+
+module Async
+  module Background
+    module Queue
+      class Store
+        class SchemaError < StandardError; end
+      end
+
+      module Schema
+        VERSION = 1
+        MIGRATION_BUSY_TIMEOUT_MS = 30_000
+        CORE_INDEXES = %w[idx_jobs_pending].freeze
+        DASHBOARD_INDEXES = %w[
+          idx_jobs_done_finished_at
+          idx_jobs_failed_finished_at
+          idx_jobs_executing_started_at
+          idx_jobs_claimed_locked_at
+        ].freeze
+        REQUIRED_INDEXES = CORE_INDEXES
+
+        module_function
+
+        def migrate!(db)
+          reject_future_version!(db)
+          return if current?(db)
+
+          enable_incremental_vacuum!(db) unless jobs_table?(db)
+
+          with_migration_timeout(db) do
+            immediate_transaction(db) do
+              reject_future_version!(db)
+              upgrade!(db) unless current?(db)
+            end
+          end
+        end
+
+        def prepare_dashboard!(db)
+          migrate!(db)
+          return if dashboard_indexes_current?(db)
+
+          with_migration_timeout(db) do
+            immediate_transaction(db) do
+              create_dashboard_indexes!(db) unless dashboard_indexes_current?(db)
+            end
+          end
+        end
+
+        def current?(db)
+          jobs_table?(db) && version(db) == VERSION && core_indexes_current?(db)
+        end
+
+        def dashboard_indexes_current?(db)
+          (DASHBOARD_INDEXES - index_names(db)).empty?
+        end
+
+        def version(db)
+          db.get_first_value(SQL::USER_VERSION).to_i
+        end
+
+        def upgrade!(db)
+          jobs_table?(db) ? upgrade_existing_database!(db) : create_current_schema!(db)
+        end
+
+        def create_current_schema!(db)
+          db.execute_batch(SQL::CREATE_SCHEMA)
+          set_version!(db, VERSION)
+        end
+
+        def upgrade_existing_database!(db)
+          add_column_unless_exists!(db, 'options', 'TEXT')
+          add_lifecycle_columns!(db)
+          backfill_finished_at!(db)
+          ensure_pending_index!(db)
+          set_version!(db, VERSION)
+        end
+
+        def add_lifecycle_columns!(db)
+          {
+            'claim_token' => 'TEXT',
+            'started_at' => 'REAL',
+            'finished_at' => 'REAL',
+            'duration_ms' => 'INTEGER',
+            'last_error_class' => 'TEXT',
+            'last_error_message' => 'TEXT'
+          }.each { |name, type| add_column_unless_exists!(db, name, type) }
+        end
+
+        def backfill_finished_at!(db)
+          db.execute(SQL::BACKFILL_FINISHED_AT)
+        end
+
+        def ensure_pending_index!(db)
+          db.execute(SQL::DROP_LEGACY_PENDING_INDEX)
+          db.execute(SQL::CREATE_PENDING_INDEX)
+        end
+
+        def create_dashboard_indexes!(db)
+          SQL::CREATE_DASHBOARD_INDEXES.each { |statement| db.execute(statement) }
+        end
+
+        def core_indexes_current?(db)
+          (CORE_INDEXES - index_names(db)).empty?
+        end
+
+        def jobs_table?(db)
+          !db.get_first_value(SQL::JOBS_TABLE_EXISTS).nil?
+        end
+
+        def index_names(db)
+          db.execute(SQL::JOB_INDEX_NAMES).map(&:first)
+        end
+
+        def add_column_unless_exists!(db, name, sql_type)
+          return if table_columns(db).include?(name)
+
+          db.execute(SQL.add_column(name, sql_type))
+        end
+
+        def table_columns(db)
+          db.execute(SQL::TABLE_INFO).map { |row| row[1] }
+        end
+
+        def reject_future_version!(db)
+          return unless version(db) > VERSION
+
+          raise Store::SchemaError,
+                "queue database schema #{version(db)} is newer than supported schema #{VERSION}"
+        end
+
+        def set_version!(db, value)
+          db.execute(SQL.user_version(value))
+        end
+
+        def enable_incremental_vacuum!(db)
+          db.execute(SQL::AUTO_VACUUM_INCREMENTAL)
+        end
+
+        def with_migration_timeout(db)
+          original_timeout = db.get_first_value(SQL::BUSY_TIMEOUT).to_i
+          db.execute(SQL.busy_timeout(MIGRATION_BUSY_TIMEOUT_MS))
+          yield
+        ensure
+          begin
+            db.execute(SQL.busy_timeout(original_timeout)) if defined?(original_timeout)
+          rescue StandardError
+            # Restoring a connection option must not hide the migration exception.
+          end
+        end
+
+        def immediate_transaction(db)
+          db.execute(SQL::BEGIN_IMMEDIATE)
+          result = yield
+          db.execute(SQL::COMMIT)
+          result
+        rescue StandardError
+          db.execute(SQL::ROLLBACK) rescue nil
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/async/background/queue/sql.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Queue
+      module SQL
+        USER_VERSION = 'PRAGMA user_version'.freeze
+        DATA_VERSION = 'PRAGMA data_version'.freeze
+        BUSY_TIMEOUT = 'PRAGMA busy_timeout'.freeze
+        TABLE_INFO = 'PRAGMA table_info(jobs)'.freeze
+        OPTIMIZE = 'PRAGMA optimize'.freeze
+        INCREMENTAL_VACUUM = 'PRAGMA incremental_vacuum'.freeze
+        AUTO_VACUUM_INCREMENTAL = 'PRAGMA auto_vacuum = INCREMENTAL'.freeze
+        BEGIN_IMMEDIATE = 'BEGIN IMMEDIATE'.freeze
+        COMMIT = 'COMMIT'.freeze
+        ROLLBACK = 'ROLLBACK'.freeze
+
+        def self.busy_timeout(milliseconds)
+          "PRAGMA busy_timeout = #{Integer(milliseconds)}"
+        end
+
+        def self.user_version(version)
+          "PRAGMA user_version = #{Integer(version)}"
+        end
+
+        def self.add_column(name, sql_type)
+          "ALTER TABLE jobs ADD COLUMN #{name} #{sql_type}"
+        end
+
+        CREATE_SCHEMA = <<~SQL.freeze
+          CREATE TABLE IF NOT EXISTS jobs (
+            id                 INTEGER PRIMARY KEY,
+            class_name         TEXT    NOT NULL,
+            args               TEXT    NOT NULL DEFAULT '[]',
+            options            TEXT,
+            status             TEXT    NOT NULL DEFAULT 'pending',
+            created_at         REAL    NOT NULL,
+            run_at             REAL    NOT NULL,
+            locked_by          INTEGER,
+            locked_at          REAL,
+            claim_token        TEXT,
+            started_at         REAL,
+            finished_at        REAL,
+            duration_ms        INTEGER,
+            last_error_class   TEXT,
+            last_error_message TEXT
+          );
+          CREATE INDEX IF NOT EXISTS idx_jobs_pending
+            ON jobs(run_at) WHERE status = 'pending';
+        SQL
+
+        INSERT_JOB = <<~SQL.freeze
+          INSERT INTO jobs (class_name, args, options, created_at, run_at)
+          VALUES (?, ?, ?, ?, ?)
+        SQL
+
+        FETCH_NEXT_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'running',
+                 locked_by   = ?,
+                 locked_at   = ?,
+                 claim_token = ?,
+                 started_at  = NULL,
+                 finished_at = NULL,
+                 duration_ms = NULL
+          WHERE  id = (
+            SELECT id FROM jobs
+            WHERE  status = 'pending' AND run_at <= ?
+            ORDER BY run_at, id
+            LIMIT 1
+          )
+          RETURNING id, class_name, args, options
+        SQL
+
+        MARK_STARTED = <<~SQL.freeze
+          UPDATE jobs
+          SET    started_at = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running' AND started_at IS NULL
+        SQL
+
+        COMPLETE_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'done',
+                 locked_by   = NULL,
+                 locked_at   = NULL,
+                 finished_at = ?,
+                 duration_ms = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        FAIL_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status             = 'failed',
+                 locked_by          = NULL,
+                 locked_at          = NULL,
+                 finished_at        = ?,
+                 duration_ms        = ?,
+                 last_error_class   = ?,
+                 last_error_message = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RETRY_STATE = <<~SQL.freeze
+          SELECT options
+          FROM jobs
+          WHERE id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        LEASE_ALIVE = <<~SQL.freeze
+          SELECT 1
+          FROM jobs
+          WHERE id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RETRY_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status             = 'pending',
+                 locked_by          = NULL,
+                 locked_at          = NULL,
+                 claim_token        = NULL,
+                 started_at         = NULL,
+                 finished_at        = NULL,
+                 duration_ms        = NULL,
+                 run_at             = ?,
+                 options            = ?,
+                 last_error_class   = ?,
+                 last_error_message = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RECOVER_WORKER = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'pending',
+                 locked_by   = NULL,
+                 locked_at   = NULL,
+                 claim_token = NULL,
+                 started_at  = NULL
+          WHERE  status = 'running' AND locked_by = ?
+        SQL
+
+        BACKFILL_FINISHED_AT = <<~SQL.freeze
+          UPDATE jobs
+          SET finished_at = created_at
+          WHERE finished_at IS NULL AND status IN ('done', 'failed')
+        SQL
+
+        CLEANUP_DONE = <<~SQL.freeze
+          DELETE FROM jobs
+          WHERE status = 'done' AND finished_at IS NOT NULL AND finished_at < ?
+        SQL
+
+        CLEANUP_FAILED = <<~SQL.freeze
+          DELETE FROM jobs
+          WHERE status = 'failed' AND finished_at IS NOT NULL AND finished_at < ?
+        SQL
+
+        NEXT_PENDING_RUN_AT = <<~SQL.freeze
+          SELECT MIN(run_at)
+          FROM jobs
+          WHERE status = 'pending'
+        SQL
+
+        JOBS_TABLE_EXISTS = <<~SQL.freeze
+          SELECT 1
+          FROM sqlite_master
+          WHERE type = 'table' AND name = 'jobs'
+          LIMIT 1
+        SQL
+
+        JOB_INDEX_NAMES = <<~SQL.freeze
+          SELECT name
+          FROM sqlite_master
+          WHERE type = 'index' AND tbl_name = 'jobs'
+        SQL
+
+        DROP_LEGACY_PENDING_INDEX = 'DROP INDEX IF EXISTS idx_jobs_status_run_at_id'.freeze
+
+        CREATE_PENDING_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_pending
+          ON jobs(run_at) WHERE status = 'pending'
+        SQL
+
+        CREATE_DONE_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_done_finished_at
+          ON jobs(finished_at DESC, id DESC)
+          WHERE status = 'done'
+        SQL
+
+        CREATE_FAILED_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_failed_finished_at
+          ON jobs(finished_at DESC, id DESC)
+          WHERE status = 'failed'
+        SQL
+
+        CREATE_EXECUTING_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_executing_started_at
+          ON jobs(started_at)
+          WHERE status = 'running' AND started_at IS NOT NULL
+        SQL
+
+        CREATE_CLAIMED_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_claimed_locked_at
+          ON jobs(locked_at)
+          WHERE status = 'running' AND started_at IS NULL
+        SQL
+
+        CREATE_DASHBOARD_INDEXES = [
+          CREATE_DONE_INDEX,
+          CREATE_FAILED_INDEX,
+          CREATE_EXECUTING_INDEX,
+          CREATE_CLAIMED_INDEX
+        ].freeze
+      end
+    end
+  end
+end