@sqlane/core 0.0.1 → 0.1.0

package/src/schema/mysql.js

@@ -0,0 +1,74 @@
+// The schema of sqlane on MySQL 8.4+. Each statement is idempotent: MySQL commits each DDL statement, so a migration
+// cut in the middle runs again from its start.
+
+/** IDs, names, keys, tokens: ASCII, compared byte by byte (case-sensitive, like PostgreSQL). */
+const ASCII = 'CHARACTER SET ascii COLLATE ascii_bin';
+
+/**
+ * @param {string} p The table prefix.
+ * @returns {import('../types.js').Migration[]}
+ */
+export function mysqlMigrations(p) {
+    return [
+        {
+            id: '0001_initial',
+            statements: [
+                `CREATE TABLE IF NOT EXISTS ${p}jobs (
+                    id VARCHAR(255) ${ASCII} NOT NULL,
+                    job_name VARCHAR(128) ${ASCII} NOT NULL,
+                    status VARCHAR(16) ${ASCII} NOT NULL,
+                    run_at DATETIME(3) NOT NULL,
+                    attempts INT NOT NULL DEFAULT 0,
+                    max_attempts INT NOT NULL,
+                    stalls INT NOT NULL DEFAULT 0,
+                    max_stalls INT NOT NULL,
+                    mutex_keys JSON NULL,
+                    enqueue_token VARCHAR(26) ${ASCII} NOT NULL,
+                    lock_token VARCHAR(26) ${ASCII} NULL,
+                    locked_by VARCHAR(255) NULL,
+                    locked_until DATETIME(3) NULL,
+                    timed_out_at DATETIME(3) NULL,
+                    last_error JSON NULL,
+                    completed_retention_ms BIGINT NOT NULL,
+                    failed_retention_ms BIGINT NOT NULL,
+                    failure_job VARCHAR(128) ${ASCII} NULL,
+                    created_at DATETIME(3) NOT NULL,
+                    started_at DATETIME(3) NULL,
+                    finished_at DATETIME(3) NULL,
+                    expires_at DATETIME(3) NULL,
+                    updated_at DATETIME(3) NOT NULL,
+                    PRIMARY KEY (id),
+                    KEY ${p}jobs_status_name_run_at (status, job_name, run_at),
+                    KEY ${p}jobs_expires_at (expires_at),
+                    CONSTRAINT ${p}jobs_status CHECK (status IN ('pending', 'running', 'completed', 'failed'))
+                ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_bin`,
+                `CREATE TABLE IF NOT EXISTS ${p}payloads (
+                    id VARCHAR(255) ${ASCII} NOT NULL,
+                    payload JSON NOT NULL,
+                    PRIMARY KEY (id)
+                ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_bin`,
+                `CREATE TABLE IF NOT EXISTS ${p}mutexes (
+                    mutex_key VARCHAR(255) ${ASCII} NOT NULL,
+                    job_id VARCHAR(255) ${ASCII} NOT NULL,
+                    lock_token VARCHAR(26) ${ASCII} NOT NULL,
+                    held_until DATETIME(3) NULL,
+                    created_at DATETIME(3) NOT NULL,
+                    PRIMARY KEY (mutex_key),
+                    KEY ${p}mutexes_job_id (job_id)
+                ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_0900_bin`,
+            ],
+        },
+    ];
+}
+
+/**
+ * @param {string} p
+ * @returns {string}
+ */
+export function mysqlMigrationsTable(p) {
+    return `CREATE TABLE IF NOT EXISTS ${p}migrations (
+        id VARCHAR(64) ${ASCII} NOT NULL,
+        applied_at DATETIME(3) NOT NULL,
+        PRIMARY KEY (id)
+    ) ENGINE = InnoDB`;
+}

package/src/schema/postgres.js

@@ -0,0 +1,67 @@
+// The schema of sqlane on PostgreSQL 15+. The migrations run in one transaction (transactional DDL).
+
+/**
+ * @param {string} p The table prefix.
+ * @returns {import('../types.js').Migration[]}
+ */
+export function postgresMigrations(p) {
+    return [
+        {
+            id: '0001_initial',
+            statements: [
+                `CREATE TABLE IF NOT EXISTS ${p}jobs (
+                    id varchar(255) COLLATE "C" NOT NULL PRIMARY KEY,
+                    job_name varchar(128) COLLATE "C" NOT NULL,
+                    status varchar(16) COLLATE "C" NOT NULL
+                        CONSTRAINT ${p}jobs_status CHECK (status IN ('pending', 'running', 'completed', 'failed')),
+                    run_at timestamptz(3) NOT NULL,
+                    attempts integer NOT NULL DEFAULT 0,
+                    max_attempts integer NOT NULL,
+                    stalls integer NOT NULL DEFAULT 0,
+                    max_stalls integer NOT NULL,
+                    mutex_keys text[] COLLATE "C",
+                    enqueue_token varchar(26) COLLATE "C" NOT NULL,
+                    lock_token varchar(26) COLLATE "C",
+                    locked_by varchar(255),
+                    locked_until timestamptz(3),
+                    timed_out_at timestamptz(3),
+                    last_error jsonb,
+                    completed_retention_ms bigint NOT NULL,
+                    failed_retention_ms bigint NOT NULL,
+                    failure_job varchar(128) COLLATE "C",
+                    created_at timestamptz(3) NOT NULL,
+                    started_at timestamptz(3),
+                    finished_at timestamptz(3),
+                    expires_at timestamptz(3),
+                    updated_at timestamptz(3) NOT NULL
+                )`,
+                `CREATE INDEX IF NOT EXISTS ${p}jobs_pending ON ${p}jobs (job_name, run_at) WHERE status = 'pending'`,
+                `CREATE INDEX IF NOT EXISTS ${p}jobs_running ON ${p}jobs (locked_until) WHERE status = 'running'`,
+                `CREATE INDEX IF NOT EXISTS ${p}jobs_expires_at ON ${p}jobs (expires_at)`,
+                `CREATE TABLE IF NOT EXISTS ${p}payloads (
+                    id varchar(255) COLLATE "C" NOT NULL PRIMARY KEY,
+                    payload jsonb NOT NULL
+                )`,
+                `CREATE TABLE IF NOT EXISTS ${p}mutexes (
+                    mutex_key varchar(255) COLLATE "C" NOT NULL PRIMARY KEY,
+                    job_id varchar(255) COLLATE "C" NOT NULL,
+                    lock_token varchar(26) COLLATE "C" NOT NULL,
+                    held_until timestamptz(3),
+                    created_at timestamptz(3) NOT NULL
+                )`,
+                `CREATE INDEX IF NOT EXISTS ${p}mutexes_job_id ON ${p}mutexes (job_id)`,
+            ],
+        },
+    ];
+}
+
+/**
+ * @param {string} p
+ * @returns {string}
+ */
+export function postgresMigrationsTable(p) {
+    return `CREATE TABLE IF NOT EXISTS ${p}migrations (
+        id varchar(64) COLLATE "C" NOT NULL PRIMARY KEY,
+        applied_at timestamptz(3) NOT NULL
+    )`;
+}

package/src/store/mysql_enqueue.js

@@ -0,0 +1,98 @@
+// Enqueue on MySQL: in the caller's transaction. MySQL has no RETURNING: the rows of this call are the rows with its
+// enqueue token.
+
+/** Rows per INSERT: 14 placeholders a row stay far below the 65,535 of MySQL. */
+const CHUNK_SIZE = 500;
+
+/**
+ * @param {Date} date
+ * @returns {string} The UTC time, as DATETIME(3) reads it.
+ */
+function toUtcDatetime(date) {
+    return date.toISOString().replace('T', ' ').replace('Z', '');
+}
+
+/**
+ * @param {import('../new_job.js').NewJob} job
+ * @returns {{ sql: string, params: unknown[] }} The run_at expression of the job.
+ */
+function runAtSql(job) {
+    return job.runAt
+        ? { sql: '?', params: [toUtcDatetime(job.runAt)] }
+        : { sql: 'UTC_TIMESTAMP(3) + INTERVAL ? MICROSECOND', params: [job.delayMs * 1000] };
+}
+
+/**
+ * @param {import('../types.js').Executor} executor
+ * @param {string} p
+ * @param {import('../new_job.js').NewJob[]} jobs
+ * @param {string} enqueueToken
+ * @returns {Promise<Set<string>>} The ids that this call created.
+ */
+async function insertChunk(executor, p, jobs, enqueueToken) {
+    const values = jobs.map(job => `(?, ?, 'pending', ${runAtSql(job).sql}, ?, ?, ?, ?, ?, ?, ?, `
+        + 'UTC_TIMESTAMP(3), UTC_TIMESTAMP(3))');
+    const params = jobs.flatMap(job => [
+        job.id,
+        job.name,
+        ...runAtSql(job).params,
+        job.maxAttempts,
+        job.maxStalls,
+        job.mutexKeys.length ? JSON.stringify(job.mutexKeys) : null,
+        enqueueToken,
+        job.completedRetentionMs,
+        job.failedRetentionMs,
+        job.failureJob,
+    ]);
+
+    // A duplicate id keeps its row: its token is not this one.
+    await executor.query(`
+        INSERT INTO ${p}jobs (id, job_name, status, run_at, max_attempts, max_stalls, mutex_keys, enqueue_token,
+            completed_retention_ms, failed_retention_ms, failure_job, created_at, updated_at)
+        VALUES ${values.join(', ')}
+        ON DUPLICATE KEY UPDATE id = id
+    `, params);
+
+    const ids = jobs.map(job => job.id);
+    const rows = await executor.query(`
+        SELECT id FROM ${p}jobs WHERE id IN (${ids.map(() => '?').join(', ')}) AND enqueue_token = ?
+    `, [...ids, enqueueToken]);
+    const created = new Set(rows.map(row => String(row.id)));
+    const createdJobs = jobs.filter(job => created.has(job.id));
+
+    if (createdJobs.length) {
+        // A payload row without its job (a manual change) belongs to the new job.
+        await executor.query(`
+            INSERT INTO ${p}payloads (id, payload) VALUES ${createdJobs.map(() => '(?, ?)').join(', ')} AS new
+            ON DUPLICATE KEY UPDATE payload = new.payload
+        `, createdJobs.flatMap(job => [job.id, job.payloadJson]));
+    }
+
+    return created;
+}
+
+/**
+ * Inserts the jobs and their payloads. Call it in a transaction.
+ * @param {import('../types.js').Executor} executor
+ * @param {string} p The table prefix.
+ * @param {import('../new_job.js').NewJob[]} jobs Their ids are distinct.
+ * @param {string} enqueueToken
+ * @returns {Promise<Set<string>>} The ids that this call created.
+ */
+export async function insertJobsMysql(executor, p, jobs, enqueueToken) {
+    const created = new Set();
+    // In the order of the ids: concurrent enqueues lock the rows in one order, without deadlocks.
+    const sortedJobs = jobs.toSorted((a, b) => (a.id < b.id ? -1 : 1));
+
+    for (let start = 0; start < sortedJobs.length; start += CHUNK_SIZE) {
+        const chunk = sortedJobs.slice(start, start + CHUNK_SIZE);
+        // oxlint-disable-next-line no-await-in-loop -- one connection: the chunks run one after the other
+        const chunkCreated = await insertChunk(executor, p, chunk, enqueueToken);
+
+        for (const id of chunkCreated) {
+            created.add(id);
+        }
+    }
+
+    return created;
+}

package/src/store/mysql_final.js

@@ -0,0 +1,424 @@
+// The final writes on MySQL: lease renewal, complete, retry, fail, release, stall recovery, sweeps. Each write of a
+// job checks its lock token: a process that lost the lease changes nothing. Each runs in a queue transaction.
+import { monotonicFactory } from 'ulid';
+import { insertJobsMysql } from './mysql_enqueue.js';
+
+/** @typedef {import('../types.js').Executor} Executor */
+/** @typedef {import('./store.js').OwnedJob} OwnedJob */
+/** @typedef {import('./store.js').FailureJobBuilder} FailureJobBuilder */
+/** @typedef {import('./store.js').FailureJobResult} FailureJobResult */
+
+const NOW = 'UTC_TIMESTAMP(3)';
+const CLEAR_LOCK = 'lock_token = NULL, locked_by = NULL, locked_until = NULL, timed_out_at = NULL';
+const enqueueToken = monotonicFactory();
+
+/**
+ * @param {number} count
+ * @returns {string}
+ */
+function placeholders(count) {
+    return Array.from({ length: count }, () => '?').join(', ');
+}
+
+/**
+ * Renews the leases of the jobs that the process runs. A row locked by another transaction is skipped: it does not
+ * block the others.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob[]} jobs
+ * @param {number} leaseMs
+ * @returns {Promise<Set<string>>} The ids whose lease was renewed.
+ */
+export async function renewLeases(executor, p, jobs, leaseMs) {
+    if (!jobs.length) {
+        return new Set();
+    }
+
+    const rows = await executor.query(`
+        SELECT id FROM ${p}jobs FORCE INDEX (PRIMARY)
+         WHERE id IN (${placeholders(jobs.length)}) AND (id, lock_token) IN (${jobs.map(() => '(?, ?)').join(', ')})
+           AND status = 'running'
+           FOR UPDATE SKIP LOCKED
+    `, [...jobs.map(job => job.id), ...jobs.flatMap(job => [job.id, job.lockToken])]);
+    const ids = rows.map(row => String(row.id));
+
+    if (ids.length) {
+        await executor.run(`
+            UPDATE ${p}jobs SET locked_until = ${NOW} + INTERVAL ? MICROSECOND WHERE id IN (${placeholders(ids.length)})
+        `, [leaseMs * 1000, ...ids]);
+    }
+
+    return new Set(ids);
+}
+
+/**
+ * Marks a timeout: the job stays running until its handler settles (`stuck` in the stats).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<boolean>} False: the lease was lost.
+ */
+export async function markTimedOut(executor, p, job) {
+    return await executor.run(`
+        UPDATE ${p}jobs SET timed_out_at = ${NOW}, updated_at = ${NOW} WHERE id = ? AND lock_token = ?
+    `, [job.id, job.lockToken]) > 0;
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<void>}
+ */
+async function deleteHolds(executor, p, job) {
+    await executor.run(`DELETE FROM ${p}mutexes WHERE job_id = ? AND lock_token = ?`, [job.id, job.lockToken]);
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {number} lingerMs The keys stay held this long. 0: deleted.
+ * @returns {Promise<void>}
+ */
+async function lingerHolds(executor, p, job, lingerMs) {
+    if (lingerMs <= 0) {
+        await deleteHolds(executor, p, job);
+
+        return;
+    }
+
+    await executor.run(`
+        UPDATE ${p}mutexes SET held_until = ${NOW} + INTERVAL ? MICROSECOND
+         WHERE job_id = ? AND lock_token = ? AND held_until IS NULL
+    `, [lingerMs * 1000, job.id, job.lockToken]);
+}
+
+/**
+ * The job is done: completed until its retention ends, without its payload; its keys are freed.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<boolean>} False: no change (the lease was lost, or an earlier try committed).
+ */
+export async function completeJob(executor, p, job) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'completed', finished_at = ${NOW},
+               expires_at = ${NOW} + INTERVAL completed_retention_ms * 1000 MICROSECOND, ${CLEAR_LOCK},
+               updated_at = ${NOW}
+         WHERE id = ? AND lock_token = ?
+    `, [job.id, job.lockToken]);
+
+    if (changed) {
+        await executor.run(`DELETE FROM ${p}payloads WHERE id = ?`, [job.id]);
+    }
+
+    await deleteHolds(executor, p, job);
+
+    return changed > 0;
+}
+
+/**
+ * The attempt failed: pending again after the delay, one more attempt used; its keys are freed.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, error: import('../error_json.js').ErrorJson }} params
+ * @returns {Promise<boolean>} False: no change.
+ */
+export async function retryJob(executor, p, job, { delayMs, error }) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'pending', run_at = ${NOW} + INTERVAL ? MICROSECOND, attempts = attempts + 1, ${CLEAR_LOCK},
+               last_error = ?, updated_at = ${NOW}
+         WHERE id = ? AND lock_token = ?
+    `, [delayMs * 1000, JSON.stringify(error), job.id, job.lockToken]);
+
+    await deleteHolds(executor, p, job);
+
+    return changed > 0;
+}
+
+/**
+ * Adds the failure job of a job that this transaction failed, under a savepoint: if the database refuses it, the job
+ * still fails, without it.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {Record<string, unknown>} row The failed job, read in this transaction.
+ * @param {{ error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<FailureJobResult>}
+ */
+async function addFailureJob(executor, p, row, { error, buildFailureJob }) {
+    let newJob;
+
+    try {
+        newJob = buildFailureJob({
+            id: String(row.id),
+            name: String(row.job_name),
+            payload: row.payload === null ? null : JSON.parse(String(row.payload)),
+            attempt: Number(row.attempts) + 1,
+            maxAttempts: Number(row.max_attempts),
+            failureJob: String(row.failure_job),
+            error,
+        });
+    } catch (buildError) {
+        return { notAdded: buildError };
+    }
+
+    await executor.run('SAVEPOINT sqlane_failure_job', []);
+
+    try {
+        await insertJobsMysql(executor, p, [newJob], enqueueToken());
+
+        return { added: newJob };
+    } catch (insertError) {
+        // MySQL rolls back only the failed statement: the job row can be there without its payload
+        await executor.run('ROLLBACK TO SAVEPOINT sqlane_failure_job', []);
+
+        return { notAdded: insertError };
+    }
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string} where The condition of the row, on `j`.
+ * @param {unknown[]} params
+ * @returns {Promise<Record<string, unknown> | undefined>} The job and its payload, locked.
+ */
+async function lockJob(executor, p, where, params) {
+    const [row] = await executor.query(`
+        SELECT j.id, j.job_name, j.lock_token, j.attempts, j.max_attempts, j.stalls, j.max_stalls, j.failure_job,
+               CAST(pl.payload AS CHAR) AS payload
+          FROM ${p}jobs j LEFT JOIN ${p}payloads pl ON pl.id = j.id
+         WHERE ${where}
+           FOR UPDATE OF j
+    `, params);
+
+    return row;
+}
+
+/**
+ * The job fails for good: failed until its retention ends, with its payload; its keys are freed; its failure job is
+ * added in the same transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ changed: boolean, failureJob: FailureJobResult | null }>}
+ */
+export async function failJob(executor, p, job, { error, buildFailureJob }) {
+    const row = await lockJob(executor, p, 'j.id = ? AND j.lock_token = ?', [job.id, job.lockToken]);
+
+    if (!row) {
+        return { changed: false, failureJob: null };
+    }
+
+    await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'failed', attempts = attempts + 1, finished_at = ${NOW},
+               expires_at = ${NOW} + INTERVAL failed_retention_ms * 1000 MICROSECOND, ${CLEAR_LOCK},
+               last_error = ?, updated_at = ${NOW}
+         WHERE id = ?
+    `, [JSON.stringify(error), job.id]);
+    await deleteHolds(executor, p, job);
+
+    return {
+        changed: true,
+        failureJob: row.failure_job === null ? null : await addFailureJob(executor, p, row, { error, buildFailureJob }),
+    };
+}
+
+/**
+ * The job is released, without an attempt used: pending again after the delay.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, lingerMs: number }} params lingerMs: its keys stay held this long (its handler still
+ *     runs); 0: they are freed.
+ * @returns {Promise<boolean>} False: no change.
+ */
+export async function releaseJob(executor, p, job, { delayMs, lingerMs }) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'pending', run_at = ${NOW} + INTERVAL ? MICROSECOND, ${CLEAR_LOCK}, updated_at = ${NOW}
+         WHERE id = ? AND lock_token = ?
+    `, [delayMs * 1000, job.id, job.lockToken]);
+
+    await lingerHolds(executor, p, job, lingerMs);
+
+    return changed > 0;
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ id: string, name: string }[]>} The running jobs whose lease ended, the oldest first.
+ */
+export async function findStaleJobs(executor, p, limit) {
+    const rows = await executor.query(`
+        SELECT id, job_name FROM ${p}jobs FORCE INDEX (${p}jobs_status_name_run_at)
+         WHERE status = 'running' AND locked_until < ${NOW}
+         ORDER BY locked_until LIMIT ?
+    `, [limit]);
+
+    return rows.map(row => ({ id: String(row.id), name: String(row.job_name) }));
+}
+
+/**
+ * A running job whose lease ended: one more stall, pending again after 30 s × 2^stalls (at most 1 h); after maxStalls,
+ * it fails for good. Its keys linger for one lease: its old handler may still run.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string} id
+ * @param {{ leaseMs: number, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} null: not stale
+ *     (any more).
+ */
+export async function recoverStaleJob(executor, p, id, { leaseMs, buildFailureJob }) {
+    const row = await lockJob(executor, p, `j.id = ? AND j.status = 'running' AND j.locked_until < ${NOW}`, [id]);
+
+    if (!row) {
+        return { result: null, failureJob: null };
+    }
+
+    const owned = { id, lockToken: String(row.lock_token) };
+    const error = { name: 'StalledError', message: 'The lease of the job ended: its process stopped or blocked' };
+    const stalls = Number(row.stalls);
+
+    await lingerHolds(executor, p, owned, leaseMs);
+
+    if (stalls < Number(row.max_stalls)) {
+        // run_at before stalls: MySQL sets the columns from left to right
+        await executor.run(`
+            UPDATE ${p}jobs
+               SET status = 'pending',
+                   run_at = ${NOW} + INTERVAL CAST(LEAST(30000000 * POW(2, stalls), 3600000000) AS UNSIGNED) MICROSECOND,
+                   stalls = stalls + 1, ${CLEAR_LOCK}, last_error = ?, updated_at = ${NOW}
+             WHERE id = ?
+        `, [JSON.stringify(error), id]);
+
+        return { result: 'released', failureJob: null };
+    }
+
+    await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'failed', stalls = stalls + 1, finished_at = ${NOW},
+               expires_at = ${NOW} + INTERVAL failed_retention_ms * 1000 MICROSECOND, ${CLEAR_LOCK},
+               last_error = ?, updated_at = ${NOW}
+         WHERE id = ?
+    `, [JSON.stringify(error), id]);
+
+    return {
+        result: 'failed',
+        failureJob: row.failure_job === null
+            ? null
+            : await addFailureJob(executor, p, row, { error, buildFailureJob }),
+    };
+}
+
+/**
+ * Deletes the ended lingers, and the holds without their running job (from a manual change, or old code).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ lingers: number, orphans: number }>}
+ */
+export async function sweepMutexes(executor, p, limit) {
+    const lingers = await executor.run(`DELETE FROM ${p}mutexes WHERE held_until < ${NOW} LIMIT ?`, [limit]);
+    const orphanKeys = (await executor.query(`
+        SELECT m.mutex_key FROM ${p}mutexes m
+         WHERE m.held_until IS NULL AND NOT EXISTS (
+             SELECT 1 FROM ${p}jobs j WHERE j.id = m.job_id AND j.status = 'running' AND j.lock_token = m.lock_token
+         )
+         LIMIT ?
+    `, [limit])).map(row => String(row.mutex_key));
+    const orphans = orphanKeys.length
+        ? await executor.run(`
+            DELETE m FROM ${p}mutexes m LEFT JOIN ${p}jobs j
+                ON j.id = m.job_id AND j.status = 'running' AND j.lock_token = m.lock_token
+             WHERE m.mutex_key IN (${placeholders(orphanKeys.length)}) AND m.held_until IS NULL AND j.id IS NULL
+        `, orphanKeys)
+        : 0;
+
+    return { lingers, orphans };
+}
+
+/**
+ * Deletes a batch of completed and failed jobs whose retention ended, with their payloads. SKIP LOCKED: processes
+ * share the work.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<number>} The deleted jobs.
+ */
+export async function deleteExpiredJobs(executor, p, limit) {
+    const ids = (await executor.query(`
+        SELECT id FROM ${p}jobs FORCE INDEX (${p}jobs_expires_at)
+         WHERE expires_at < ${NOW} ORDER BY expires_at LIMIT ?
+           FOR UPDATE SKIP LOCKED
+    `, [limit])).map(row => String(row.id));
+
+    if (!ids.length) {
+        return 0;
+    }
+
+    const deleted = await executor.run(`DELETE FROM ${p}jobs WHERE id IN (${placeholders(ids.length)})`, ids);
+
+    await executor.run(`DELETE FROM ${p}payloads WHERE id IN (${placeholders(ids.length)})`, ids);
+
+    return deleted;
+}
+
+/**
+ * For drain({ includeDelayed }): the pending jobs of the names are due now.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} names
+ * @returns {Promise<void>}
+ */
+export async function makeDue(executor, p, names) {
+    if (!names.length) {
+        return;
+    }
+
+    await executor.run(`
+        UPDATE ${p}jobs SET run_at = ${NOW}
+         WHERE status = 'pending' AND job_name IN (${placeholders(names.length)}) AND run_at > ${NOW}
+    `, names);
+}
+
+/**
+ * The counts per job name, for alerts such as "queue stuck". Reads the status index only.
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<import('../types.js').JobStats[]>}
+ */
+export async function getStats(executor, p) {
+    const rows = await executor.query(`
+        SELECT job_name,
+               SUM(status = 'pending') AS pending,
+               SUM(status = 'pending' AND run_at <= ${NOW}) AS due,
+               SUM(status = 'running') AS running,
+               SUM(status = 'running' AND timed_out_at IS NOT NULL) AS stuck,
+               SUM(status = 'failed') AS failed,
+               TIMESTAMPDIFF(MICROSECOND, MIN(IF(status = 'pending' AND run_at <= ${NOW}, run_at, NULL)), ${NOW})
+                   DIV 1000 AS oldest_due_age_ms
+          FROM ${p}jobs
+         WHERE status IN ('pending', 'running', 'failed')
+         GROUP BY job_name
+         ORDER BY job_name
+    `, []);
+
+    return rows.map(row => ({
+        name: String(row.job_name),
+        pending: Number(row.pending),
+        due: Number(row.due),
+        running: Number(row.running),
+        stuck: Number(row.stuck),
+        failed: Number(row.failed),
+        oldestDueAgeMs: row.oldest_due_age_ms === null ? null : Number(row.oldest_due_age_ms),
+    }));
+}