async-background 0.7.1 → 0.7.2

data/lib/async/background/queue/schema.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative 'sql'
+
+module Async
+  module Background
+    module Queue
+      class Store
+        class SchemaError < StandardError; end
+      end
+
+      module Schema
+        VERSION = 1
+        MIGRATION_BUSY_TIMEOUT_MS = 30_000
+        CORE_INDEXES = %w[idx_jobs_pending].freeze
+        DASHBOARD_INDEXES = %w[idx_jobs_done_finished_at idx_jobs_failed_finished_at idx_jobs_running].freeze
+        REQUIRED_INDEXES = CORE_INDEXES
+
+        module_function
+
+        def migrate!(db)
+          reject_future_version!(db)
+          return if current?(db)
+
+          enable_incremental_vacuum!(db) unless jobs_table?(db)
+
+          with_migration_timeout(db) do
+            immediate_transaction(db) do
+              reject_future_version!(db)
+              upgrade!(db) unless current?(db)
+            end
+          end
+        end
+
+        def prepare_dashboard!(db)
+          migrate!(db)
+          return if dashboard_indexes_current?(db)
+
+          with_migration_timeout(db) do
+            immediate_transaction(db) do
+              create_dashboard_indexes!(db) unless dashboard_indexes_current?(db)
+            end
+          end
+        end
+
+        def current?(db)
+          jobs_table?(db) && version(db) == VERSION && core_indexes_current?(db)
+        end
+
+        def dashboard_indexes_current?(db)
+          (DASHBOARD_INDEXES - index_names(db)).empty?
+        end
+
+        def version(db)
+          db.get_first_value(SQL::USER_VERSION).to_i
+        end
+
+        def upgrade!(db)
+          jobs_table?(db) ? upgrade_existing_database!(db) : create_current_schema!(db)
+        end
+
+        def create_current_schema!(db)
+          db.execute_batch(SQL::CREATE_SCHEMA)
+          set_version!(db, VERSION)
+        end
+
+        def upgrade_existing_database!(db)
+          add_column_unless_exists!(db, 'options', 'TEXT')
+          add_lifecycle_columns!(db)
+          backfill_finished_at!(db)
+          ensure_pending_index!(db)
+          set_version!(db, VERSION)
+        end
+
+        def add_lifecycle_columns!(db)
+          {
+            'claim_token' => 'TEXT',
+            'started_at' => 'REAL',
+            'finished_at' => 'REAL',
+            'duration_ms' => 'INTEGER',
+            'last_error_class' => 'TEXT',
+            'last_error_message' => 'TEXT'
+          }.each { |name, type| add_column_unless_exists!(db, name, type) }
+        end
+
+        def backfill_finished_at!(db)
+          db.execute(SQL::BACKFILL_FINISHED_AT)
+        end
+
+        def ensure_pending_index!(db)
+          db.execute(SQL::DROP_LEGACY_PENDING_INDEX)
+          db.execute(SQL::CREATE_PENDING_INDEX)
+        end
+
+        def create_dashboard_indexes!(db)
+          SQL::CREATE_DASHBOARD_INDEXES.each { |statement| db.execute(statement) }
+        end
+
+        def core_indexes_current?(db)
+          (CORE_INDEXES - index_names(db)).empty?
+        end
+
+        def jobs_table?(db)
+          !db.get_first_value(SQL::JOBS_TABLE_EXISTS).nil?
+        end
+
+        def index_names(db)
+          db.execute(SQL::JOB_INDEX_NAMES).map(&:first)
+        end
+
+        def add_column_unless_exists!(db, name, sql_type)
+          return if table_columns(db).include?(name)
+
+          db.execute(SQL.add_column(name, sql_type))
+        end
+
+        def table_columns(db)
+          db.execute(SQL::TABLE_INFO).map { |row| row[1] }
+        end
+
+        def reject_future_version!(db)
+          return unless version(db) > VERSION
+
+          raise Store::SchemaError,
+                "queue database schema #{version(db)} is newer than supported schema #{VERSION}"
+        end
+
+        def set_version!(db, value)
+          db.execute(SQL.user_version(value))
+        end
+
+        def enable_incremental_vacuum!(db)
+          db.execute(SQL::AUTO_VACUUM_INCREMENTAL)
+        end
+
+        def with_migration_timeout(db)
+          original_timeout = db.get_first_value(SQL::BUSY_TIMEOUT).to_i
+          db.execute(SQL.busy_timeout(MIGRATION_BUSY_TIMEOUT_MS))
+          yield
+        ensure
+          begin
+            db.execute(SQL.busy_timeout(original_timeout)) if defined?(original_timeout)
+          rescue StandardError
+            # Restoring a connection option must not hide the migration exception.
+          end
+        end
+
+        def immediate_transaction(db)
+          db.execute(SQL::BEGIN_IMMEDIATE)
+          result = yield
+          db.execute(SQL::COMMIT)
+          result
+        rescue StandardError
+          db.execute(SQL::ROLLBACK) rescue nil
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/async/background/queue/sql.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module Async
+  module Background
+    module Queue
+      module SQL
+        USER_VERSION = 'PRAGMA user_version'.freeze
+        DATA_VERSION = 'PRAGMA data_version'.freeze
+        BUSY_TIMEOUT = 'PRAGMA busy_timeout'.freeze
+        TABLE_INFO = 'PRAGMA table_info(jobs)'.freeze
+        OPTIMIZE = 'PRAGMA optimize'.freeze
+        INCREMENTAL_VACUUM = 'PRAGMA incremental_vacuum'.freeze
+        AUTO_VACUUM_INCREMENTAL = 'PRAGMA auto_vacuum = INCREMENTAL'.freeze
+        BEGIN_IMMEDIATE = 'BEGIN IMMEDIATE'.freeze
+        COMMIT = 'COMMIT'.freeze
+        ROLLBACK = 'ROLLBACK'.freeze
+
+        def self.busy_timeout(milliseconds)
+          "PRAGMA busy_timeout = #{Integer(milliseconds)}"
+        end
+
+        def self.user_version(version)
+          "PRAGMA user_version = #{Integer(version)}"
+        end
+
+        def self.add_column(name, sql_type)
+          "ALTER TABLE jobs ADD COLUMN #{name} #{sql_type}"
+        end
+
+        CREATE_SCHEMA = <<~SQL.freeze
+          CREATE TABLE IF NOT EXISTS jobs (
+            id                 INTEGER PRIMARY KEY,
+            class_name         TEXT    NOT NULL,
+            args               TEXT    NOT NULL DEFAULT '[]',
+            options            TEXT,
+            status             TEXT    NOT NULL DEFAULT 'pending',
+            created_at         REAL    NOT NULL,
+            run_at             REAL    NOT NULL,
+            locked_by          INTEGER,
+            locked_at          REAL,
+            claim_token        TEXT,
+            started_at         REAL,
+            finished_at        REAL,
+            duration_ms        INTEGER,
+            last_error_class   TEXT,
+            last_error_message TEXT
+          );
+          CREATE INDEX IF NOT EXISTS idx_jobs_pending
+            ON jobs(run_at) WHERE status = 'pending';
+        SQL
+
+        INSERT_JOB = <<~SQL.freeze
+          INSERT INTO jobs (class_name, args, options, created_at, run_at)
+          VALUES (?, ?, ?, ?, ?)
+        SQL
+
+        FETCH_NEXT_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'running',
+                 locked_by   = ?,
+                 locked_at   = ?,
+                 claim_token = ?,
+                 started_at  = NULL,
+                 finished_at = NULL,
+                 duration_ms = NULL
+          WHERE  id = (
+            SELECT id FROM jobs
+            WHERE  status = 'pending' AND run_at <= ?
+            ORDER BY run_at, id
+            LIMIT 1
+          )
+          RETURNING id, class_name, args, options
+        SQL
+
+        MARK_STARTED = <<~SQL.freeze
+          UPDATE jobs
+          SET    started_at = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running' AND started_at IS NULL
+        SQL
+
+        COMPLETE_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'done',
+                 locked_by   = NULL,
+                 locked_at   = NULL,
+                 finished_at = ?,
+                 duration_ms = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        FAIL_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status             = 'failed',
+                 locked_by          = NULL,
+                 locked_at          = NULL,
+                 finished_at        = ?,
+                 duration_ms        = ?,
+                 last_error_class   = ?,
+                 last_error_message = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RETRY_STATE = <<~SQL.freeze
+          SELECT options
+          FROM jobs
+          WHERE id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        LEASE_ALIVE = <<~SQL.freeze
+          SELECT 1
+          FROM jobs
+          WHERE id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RETRY_JOB = <<~SQL.freeze
+          UPDATE jobs
+          SET    status             = 'pending',
+                 locked_by          = NULL,
+                 locked_at          = NULL,
+                 claim_token        = NULL,
+                 started_at         = NULL,
+                 finished_at        = NULL,
+                 duration_ms        = NULL,
+                 run_at             = ?,
+                 options            = ?,
+                 last_error_class   = ?,
+                 last_error_message = ?
+          WHERE  id = ? AND claim_token = ? AND status = 'running'
+        SQL
+
+        RECOVER_WORKER = <<~SQL.freeze
+          UPDATE jobs
+          SET    status      = 'pending',
+                 locked_by   = NULL,
+                 locked_at   = NULL,
+                 claim_token = NULL,
+                 started_at  = NULL
+          WHERE  status = 'running' AND locked_by = ?
+        SQL
+
+        BACKFILL_FINISHED_AT = <<~SQL.freeze
+          UPDATE jobs
+          SET finished_at = created_at
+          WHERE finished_at IS NULL AND status IN ('done', 'failed')
+        SQL
+
+        CLEANUP_DONE = <<~SQL.freeze
+          DELETE FROM jobs
+          WHERE status = 'done' AND finished_at IS NOT NULL AND finished_at < ?
+        SQL
+
+        CLEANUP_FAILED = <<~SQL.freeze
+          DELETE FROM jobs
+          WHERE status = 'failed' AND finished_at IS NOT NULL AND finished_at < ?
+        SQL
+
+        NEXT_PENDING_RUN_AT = <<~SQL.freeze
+          SELECT MIN(run_at)
+          FROM jobs
+          WHERE status = 'pending'
+        SQL
+
+        JOBS_TABLE_EXISTS = <<~SQL.freeze
+          SELECT 1
+          FROM sqlite_master
+          WHERE type = 'table' AND name = 'jobs'
+          LIMIT 1
+        SQL
+
+        JOB_INDEX_NAMES = <<~SQL.freeze
+          SELECT name
+          FROM sqlite_master
+          WHERE type = 'index' AND tbl_name = 'jobs'
+        SQL
+
+        DROP_LEGACY_PENDING_INDEX = 'DROP INDEX IF EXISTS idx_jobs_status_run_at_id'.freeze
+
+        CREATE_PENDING_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_pending
+          ON jobs(run_at) WHERE status = 'pending'
+        SQL
+
+        CREATE_DONE_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_done_finished_at
+          ON jobs(finished_at DESC, id DESC)
+          WHERE status = 'done'
+        SQL
+
+        CREATE_FAILED_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_failed_finished_at
+          ON jobs(finished_at DESC, id DESC)
+          WHERE status = 'failed'
+        SQL
+
+        CREATE_RUNNING_INDEX = <<~SQL.freeze
+          CREATE INDEX IF NOT EXISTS idx_jobs_running
+          ON jobs(locked_at)
+          WHERE status = 'running'
+        SQL
+
+        CREATE_DASHBOARD_INDEXES = [CREATE_DONE_INDEX, CREATE_FAILED_INDEX, CREATE_RUNNING_INDEX].freeze
+      end
+    end
+  end
+end