@sqlane/core 0.0.1 → 0.1.0

package/src/store/postgres_final.js

@@ -0,0 +1,407 @@
+// The final writes on PostgreSQL: lease renewal, complete, retry, fail, release, stall recovery, sweeps. Each write of
+// a job checks its lock token: a process that lost the lease changes nothing. Each runs in a queue transaction.
+import { monotonicFactory } from 'ulid';
+import { insertJobsPostgres } from './postgres_enqueue.js';
+
+/** @typedef {import('../types.js').Executor} Executor */
+/** @typedef {import('./store.js').OwnedJob} OwnedJob */
+/** @typedef {import('./store.js').FailureJobBuilder} FailureJobBuilder */
+/** @typedef {import('./store.js').FailureJobResult} FailureJobResult */
+
+const NOW = 'clock_timestamp()';
+/**
+ * A due time from now, truncated to the ms of the column: timestamptz(3) rounds, and a rounded-up time is not due.
+ * @param {string} offset An interval.
+ * @returns {string}
+ */
+const runAt = offset => `date_trunc('milliseconds', ${NOW} + ${offset})`;
+const MS = 'interval \'1 millisecond\'';
+const CLEAR_LOCK = 'lock_token = NULL, locked_by = NULL, locked_until = NULL, timed_out_at = NULL';
+const enqueueToken = monotonicFactory();
+
+/**
+ * Renews the leases of the jobs that the process runs. A row locked by another transaction is skipped: it does not
+ * block the others.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob[]} jobs
+ * @param {number} leaseMs
+ * @returns {Promise<Set<string>>} The ids whose lease was renewed.
+ */
+export async function renewLeases(executor, p, jobs, leaseMs) {
+    if (!jobs.length) {
+        return new Set();
+    }
+
+    const rows = await executor.query(`
+        WITH locked AS (
+            SELECT j.id FROM ${p}jobs j
+              JOIN unnest($1::text[], $2::text[]) AS own(id, lock_token)
+                ON own.id = j.id AND own.lock_token = j.lock_token
+             WHERE j.status = 'running'
+               FOR UPDATE OF j SKIP LOCKED
+        )
+        UPDATE ${p}jobs j SET locked_until = ${NOW} + $3 * ${MS} FROM locked WHERE j.id = locked.id
+        RETURNING j.id
+    `, [jobs.map(job => job.id), jobs.map(job => job.lockToken), leaseMs]);
+
+    return new Set(rows.map(row => String(row.id)));
+}
+
+/**
+ * Marks a timeout: the job stays running until its handler settles (`stuck` in the stats).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<boolean>} False: the lease was lost.
+ */
+export async function markTimedOut(executor, p, job) {
+    return await executor.run(`
+        UPDATE ${p}jobs SET timed_out_at = ${NOW}, updated_at = ${NOW} WHERE id = $1 AND lock_token = $2
+    `, [job.id, job.lockToken]) > 0;
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<void>}
+ */
+async function deleteHolds(executor, p, job) {
+    await executor.run(`DELETE FROM ${p}mutexes WHERE job_id = $1 AND lock_token = $2`, [job.id, job.lockToken]);
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {number} lingerMs The keys stay held this long. 0: deleted.
+ * @returns {Promise<void>}
+ */
+async function lingerHolds(executor, p, job, lingerMs) {
+    if (lingerMs <= 0) {
+        await deleteHolds(executor, p, job);
+
+        return;
+    }
+
+    await executor.run(`
+        UPDATE ${p}mutexes SET held_until = ${NOW} + $3 * ${MS}
+         WHERE job_id = $1 AND lock_token = $2 AND held_until IS NULL
+    `, [job.id, job.lockToken, lingerMs]);
+}
+
+/**
+ * The job is done: completed until its retention ends, without its payload; its keys are freed.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<boolean>} False: no change (the lease was lost, or an earlier try committed).
+ */
+export async function completeJob(executor, p, job) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'completed', finished_at = ${NOW}, expires_at = ${NOW} + completed_retention_ms * ${MS},
+               ${CLEAR_LOCK}, updated_at = ${NOW}
+         WHERE id = $1 AND lock_token = $2
+    `, [job.id, job.lockToken]);
+
+    if (changed) {
+        await executor.run(`DELETE FROM ${p}payloads WHERE id = $1`, [job.id]);
+    }
+
+    await deleteHolds(executor, p, job);
+
+    return changed > 0;
+}
+
+/**
+ * The attempt failed: pending again after the delay, one more attempt used; its keys are freed.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, error: import('../error_json.js').ErrorJson }} params
+ * @returns {Promise<boolean>} False: no change.
+ */
+export async function retryJob(executor, p, job, { delayMs, error }) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'pending', run_at = ${runAt(`$3 * ${MS}`)}, attempts = attempts + 1, ${CLEAR_LOCK},
+               last_error = $4::jsonb, updated_at = ${NOW}
+         WHERE id = $1 AND lock_token = $2
+    `, [job.id, job.lockToken, delayMs, JSON.stringify(error)]);
+
+    await deleteHolds(executor, p, job);
+
+    return changed > 0;
+}
+
+/**
+ * Adds the failure job of a job that this transaction failed, under a savepoint: if the database refuses it, the job
+ * still fails, without it.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {Record<string, unknown>} row The failed job, read in this transaction.
+ * @param {{ error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<FailureJobResult>}
+ */
+async function addFailureJob(executor, p, row, { error, buildFailureJob }) {
+    let newJob;
+
+    try {
+        newJob = buildFailureJob({
+            id: String(row.id),
+            name: String(row.job_name),
+            payload: row.payload === null ? null : JSON.parse(String(row.payload)),
+            attempt: Number(row.attempts) + 1,
+            maxAttempts: Number(row.max_attempts),
+            failureJob: String(row.failure_job),
+            error,
+        });
+    } catch (buildError) {
+        return { notAdded: buildError };
+    }
+
+    await executor.run('SAVEPOINT sqlane_failure_job', []);
+
+    try {
+        await insertJobsPostgres(executor, p, [newJob], enqueueToken());
+        await executor.run('RELEASE SAVEPOINT sqlane_failure_job', []);
+
+        return { added: newJob };
+    } catch (insertError) {
+        await executor.run('ROLLBACK TO SAVEPOINT sqlane_failure_job', []);
+
+        return { notAdded: insertError };
+    }
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string} where The condition of the row, on `j`.
+ * @param {unknown[]} params
+ * @returns {Promise<Record<string, unknown> | undefined>} The job and its payload, locked.
+ */
+async function lockJob(executor, p, where, params) {
+    const [row] = await executor.query(`
+        SELECT j.id, j.job_name, j.lock_token, j.attempts, j.max_attempts, j.stalls, j.max_stalls, j.failure_job,
+               pl.payload::text AS payload
+          FROM ${p}jobs j LEFT JOIN ${p}payloads pl ON pl.id = j.id
+         WHERE ${where}
+           FOR UPDATE OF j
+    `, params);
+
+    return row;
+}
+
+/**
+ * The job fails for good: failed until its retention ends, with its payload; its keys are freed; its failure job is
+ * added in the same transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ changed: boolean, failureJob: FailureJobResult | null }>}
+ */
+export async function failJob(executor, p, job, { error, buildFailureJob }) {
+    const row = await lockJob(executor, p, 'j.id = $1 AND j.lock_token = $2', [job.id, job.lockToken]);
+
+    if (!row) {
+        return { changed: false, failureJob: null };
+    }
+
+    await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'failed', attempts = attempts + 1, finished_at = ${NOW},
+               expires_at = ${NOW} + failed_retention_ms * ${MS}, ${CLEAR_LOCK}, last_error = $2::jsonb,
+               updated_at = ${NOW}
+         WHERE id = $1
+    `, [job.id, JSON.stringify(error)]);
+    await deleteHolds(executor, p, job);
+
+    return {
+        changed: true,
+        failureJob: row.failure_job === null ? null : await addFailureJob(executor, p, row, { error, buildFailureJob }),
+    };
+}
+
+/**
+ * The job is released, without an attempt used: pending again after the delay.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, lingerMs: number }} params lingerMs: its keys stay held this long (its handler still
+ *     runs); 0: they are freed.
+ * @returns {Promise<boolean>} False: no change.
+ */
+export async function releaseJob(executor, p, job, { delayMs, lingerMs }) {
+    const changed = await executor.run(`
+        UPDATE ${p}jobs SET status = 'pending', run_at = ${runAt(`$3 * ${MS}`)}, ${CLEAR_LOCK}, updated_at = ${NOW}
+         WHERE id = $1 AND lock_token = $2
+    `, [job.id, job.lockToken, delayMs]);
+
+    await lingerHolds(executor, p, job, lingerMs);
+
+    return changed > 0;
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ id: string, name: string }[]>} The running jobs whose lease ended, the oldest first.
+ */
+export async function findStaleJobs(executor, p, limit) {
+    const rows = await executor.query(`
+        SELECT id, job_name FROM ${p}jobs
+         WHERE status = 'running' AND locked_until < ${NOW}
+         ORDER BY locked_until LIMIT $1
+    `, [limit]);
+
+    return rows.map(row => ({ id: String(row.id), name: String(row.job_name) }));
+}
+
+/**
+ * A running job whose lease ended: one more stall, pending again after 30 s × 2^stalls (at most 1 h); after maxStalls,
+ * it fails for good. Its keys linger for one lease: its old handler may still run.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string} id
+ * @param {{ leaseMs: number, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} null: not stale
+ *     (any more).
+ */
+export async function recoverStaleJob(executor, p, id, { leaseMs, buildFailureJob }) {
+    const row = await lockJob(executor, p, `j.id = $1 AND j.status = 'running' AND j.locked_until < ${NOW}`, [id]);
+
+    if (!row) {
+        return { result: null, failureJob: null };
+    }
+
+    const error = { name: 'StalledError', message: 'The lease of the job ended: its process stopped or blocked' };
+
+    await lingerHolds(executor, p, { id, lockToken: String(row.lock_token) }, leaseMs);
+
+    if (Number(row.stalls) < Number(row.max_stalls)) {
+        await executor.run(`
+            UPDATE ${p}jobs
+               SET status = 'pending', run_at = ${runAt(`LEAST(30000 * power(2, stalls), 3600000) * ${MS}`)},
+                   stalls = stalls + 1, ${CLEAR_LOCK}, last_error = $2::jsonb, updated_at = ${NOW}
+             WHERE id = $1
+        `, [id, JSON.stringify(error)]);
+
+        return { result: 'released', failureJob: null };
+    }
+
+    await executor.run(`
+        UPDATE ${p}jobs
+           SET status = 'failed', stalls = stalls + 1, finished_at = ${NOW},
+               expires_at = ${NOW} + failed_retention_ms * ${MS}, ${CLEAR_LOCK}, last_error = $2::jsonb,
+               updated_at = ${NOW}
+         WHERE id = $1
+    `, [id, JSON.stringify(error)]);
+
+    return {
+        result: 'failed',
+        failureJob: row.failure_job === null ? null : await addFailureJob(executor, p, row, { error, buildFailureJob }),
+    };
+}
+
+/**
+ * Deletes the ended lingers, and the holds without their running job (from a manual change, or old code).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ lingers: number, orphans: number }>}
+ */
+export async function sweepMutexes(executor, p, limit) {
+    const lingers = await executor.run(`
+        DELETE FROM ${p}mutexes WHERE mutex_key IN (
+            SELECT mutex_key FROM ${p}mutexes WHERE held_until < ${NOW} LIMIT $1 FOR UPDATE SKIP LOCKED
+        )
+    `, [limit]);
+    const orphans = await executor.run(`
+        DELETE FROM ${p}mutexes WHERE mutex_key IN (
+            SELECT m.mutex_key FROM ${p}mutexes m
+             WHERE m.held_until IS NULL AND NOT EXISTS (
+                 SELECT 1 FROM ${p}jobs j WHERE j.id = m.job_id AND j.status = 'running' AND j.lock_token = m.lock_token
+             )
+             LIMIT $1 FOR UPDATE SKIP LOCKED
+        )
+    `, [limit]);
+
+    return { lingers, orphans };
+}
+
+/**
+ * Deletes a batch of completed and failed jobs whose retention ended, with their payloads. SKIP LOCKED: processes
+ * share the work.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<number>} The deleted jobs.
+ */
+export async function deleteExpiredJobs(executor, p, limit) {
+    const rows = await executor.query(`
+        WITH expired AS (
+            SELECT id FROM ${p}jobs WHERE expires_at < ${NOW} ORDER BY expires_at LIMIT $1 FOR UPDATE SKIP LOCKED
+        ),
+        deleted AS (
+            DELETE FROM ${p}jobs j USING expired WHERE j.id = expired.id RETURNING j.id
+        ),
+        payloads AS (
+            DELETE FROM ${p}payloads pl USING deleted WHERE pl.id = deleted.id
+        )
+        SELECT count(*) AS count FROM deleted
+    `, [limit]);
+
+    return Number(rows[0].count);
+}
+
+/**
+ * For drain({ includeDelayed }): the pending jobs of the names are due now.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} names
+ * @returns {Promise<void>}
+ */
+export async function makeDue(executor, p, names) {
+    await executor.run(`
+        UPDATE ${p}jobs SET run_at = ${runAt('interval \'0\'')}
+         WHERE status = 'pending' AND job_name = ANY($1::text[]) AND run_at > ${NOW}
+    `, [names]);
+}
+
+/**
+ * The counts per job name, for alerts such as "queue stuck".
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<import('../types.js').JobStats[]>}
+ */
+export async function getStats(executor, p) {
+    const rows = await executor.query(`
+        SELECT job_name,
+               count(*) FILTER (WHERE status = 'pending') AS pending,
+               count(*) FILTER (WHERE status = 'pending' AND run_at <= ${NOW}) AS due,
+               count(*) FILTER (WHERE status = 'running') AS running,
+               count(*) FILTER (WHERE status = 'running' AND timed_out_at IS NOT NULL) AS stuck,
+               count(*) FILTER (WHERE status = 'failed') AS failed,
+               (EXTRACT(EPOCH FROM ${NOW} - min(run_at) FILTER (WHERE status = 'pending' AND run_at <= ${NOW}))
+                   * 1000)::bigint AS oldest_due_age_ms
+          FROM ${p}jobs
+         WHERE status IN ('pending', 'running', 'failed')
+         GROUP BY job_name
+         ORDER BY job_name
+    `, []);
+
+    return rows.map(row => ({
+        name: String(row.job_name),
+        pending: Number(row.pending),
+        due: Number(row.due),
+        running: Number(row.running),
+        stuck: Number(row.stuck),
+        failed: Number(row.failed),
+        oldestDueAgeMs: row.oldest_due_age_ms === null ? null : Number(row.oldest_due_age_ms),
+    }));
+}

package/src/store/postgres_worker.js

@@ -0,0 +1,224 @@
+// The worker side of the store on PostgreSQL: probe, candidates, claim. Times are computed in SQL.
+
+/** @typedef {import('../types.js').Executor} Executor */
+/** @typedef {import('./store.js').ProbeRow} ProbeRow */
+/** @typedef {import('./store.js').Candidate} Candidate */
+/** @typedef {import('./store.js').ClaimParams} ClaimParams */
+/** @typedef {import('./store.js').ClaimedJob} ClaimedJob */
+
+/** The due jobs of each keyed name that the FIFO of the mutex keys reads */
+export const MUTEX_WINDOW_SIZE = 500;
+
+/**
+ * @param {string} column
+ * @returns {string} The epoch milliseconds of a time column.
+ */
+const epochMs = column => `(EXTRACT(EPOCH FROM ${column}) * 1000)::bigint`;
+const HELD = '(m.held_until IS NULL OR m.held_until >= clock_timestamp())';
+
+/**
+ * The next due time of each job name with pending jobs. PostgreSQL has no loose index scan: a recursive CTE skips from
+ * one pending name to the next on the partial index, two index probes per pending name.
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<ProbeRow[]>}
+ */
+export async function probe(executor, p) {
+    const rows = await executor.query(`
+        WITH RECURSIVE names AS (
+            (SELECT job_name FROM ${p}jobs WHERE status = 'pending' ORDER BY job_name LIMIT 1)
+            UNION ALL
+            SELECT (
+                SELECT j.job_name FROM ${p}jobs j
+                 WHERE j.status = 'pending' AND j.job_name > names.job_name ORDER BY j.job_name LIMIT 1
+            ) FROM names WHERE names.job_name IS NOT NULL
+        )
+        SELECT names.job_name,
+               (EXTRACT(EPOCH FROM (
+                   SELECT j.run_at FROM ${p}jobs j
+                    WHERE j.status = 'pending' AND j.job_name = names.job_name ORDER BY j.run_at LIMIT 1
+               ) - clock_timestamp()) * 1000)::bigint AS delay_ms
+          FROM names WHERE names.job_name IS NOT NULL
+    `, []);
+
+    return rows.map(row => ({ name: String(row.job_name), delayMs: Number(row.delay_ms) }));
+}
+
+/**
+ * The due jobs that can start now, oldest first: at most `limit` of each name. A job of a keyed name waits if one of
+ * its keys is held, or an older due job of a keyed name has it (FIFO per key over the window of the keyed names).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {Map<string, number>} limitsByName
+ * @param {string[]} keyedNames The handled names whose definition has mutex keys.
+ * @returns {Promise<Candidate[]>}
+ */
+export async function findCandidates(executor, p, limitsByName, keyedNames) {
+    const keyed = new Set(keyedNames);
+    const plainNames = [...limitsByName.keys()].filter(name => !keyed.has(name));
+    const keyedRequested = [...limitsByName.keys()].filter(name => keyed.has(name));
+    /** @type {Record<string, unknown>[]} */
+    const rows = [];
+
+    if (plainNames.length) {
+        rows.push(...await executor.query(`
+            SELECT due.id, due.job_name, ${epochMs('due.run_at')} AS run_at_ms
+              FROM unnest($1::text[], $2::integer[]) AS request(name, max_jobs)
+             CROSS JOIN LATERAL (
+                 SELECT id, job_name, run_at FROM ${p}jobs
+                  WHERE status = 'pending' AND job_name = request.name AND run_at <= clock_timestamp()
+                  ORDER BY run_at, id LIMIT request.max_jobs
+             ) AS due
+        `, [plainNames, plainNames.map(name => limitsByName.get(name))]));
+    }
+
+    if (keyedRequested.length) {
+        rows.push(...await executor.query(`
+            WITH w AS (
+                SELECT due.* FROM unnest($1::text[]) AS keyed(name)
+                 CROSS JOIN LATERAL (
+                     SELECT id, job_name, run_at, mutex_keys FROM ${p}jobs
+                      WHERE status = 'pending' AND job_name = keyed.name AND run_at <= clock_timestamp()
+                      ORDER BY run_at, id LIMIT ${MUTEX_WINDOW_SIZE}
+                 ) AS due
+            ),
+            wk AS (
+                SELECT w.id, k.mutex_key,
+                       ROW_NUMBER() OVER (PARTITION BY k.mutex_key ORDER BY w.run_at, w.id) AS key_rank
+                  FROM w CROSS JOIN LATERAL unnest(w.mutex_keys) AS k(mutex_key)
+            ),
+            busy AS (
+                SELECT wk.mutex_key FROM wk JOIN ${p}mutexes m ON m.mutex_key = wk.mutex_key
+                 WHERE wk.key_rank = 1 AND ${HELD}
+            ),
+            blocked AS (
+                SELECT id FROM wk WHERE key_rank > 1
+                 UNION SELECT wk.id FROM wk JOIN busy ON busy.mutex_key = wk.mutex_key
+            ),
+            eligible AS (
+                SELECT w.id, w.job_name, w.run_at,
+                       ROW_NUMBER() OVER (PARTITION BY w.job_name ORDER BY w.run_at, w.id) AS name_rank
+                  FROM w WHERE NOT EXISTS (SELECT 1 FROM blocked b WHERE b.id = w.id)
+            )
+            SELECT e.id, e.job_name, ${epochMs('e.run_at')} AS run_at_ms
+              FROM eligible e JOIN unnest($2::text[], $3::integer[]) AS request(name, max_jobs)
+                ON request.name = e.job_name AND e.name_rank <= request.max_jobs
+        `, [keyedNames, keyedRequested, keyedRequested.map(name => limitsByName.get(name))]));
+    }
+
+    return rows
+        .map(row => ({ id: String(row.id), name: String(row.job_name), runAtMs: Number(row.run_at_ms) }))
+        .toSorted((a, b) => a.runAtMs - b.runAtMs || (a.id < b.id ? -1 : 1));
+}
+
+/**
+ * Takes the keys of the picked jobs, all or none per job. A held key stays with its holder; an ended linger is taken
+ * over. Keys are inserted in ascending order: no deadlocks between claims.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {{ id: string, mutexKeys: readonly string[] }[]} jobs The picked keyed jobs: their keys are distinct.
+ * @param {string} lockToken
+ * @returns {Promise<Set<string>>} The ids of the jobs that got all their keys.
+ */
+async function takeKeys(executor, p, jobs, lockToken) {
+    const holds = jobs
+        .flatMap(job => job.mutexKeys.map(key => ({ key, id: job.id })))
+        .toSorted((a, b) => (a.key < b.key ? -1 : 1));
+    const acquired = new Set((await executor.query(`
+        INSERT INTO ${p}mutexes AS m (mutex_key, job_id, lock_token, held_until, created_at)
+        SELECT hold.mutex_key, hold.job_id, $3, NULL, clock_timestamp()
+          FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS hold(mutex_key, job_id, position)
+         ORDER BY hold.position
+        ON CONFLICT (mutex_key) DO UPDATE
+           SET job_id = EXCLUDED.job_id, lock_token = EXCLUDED.lock_token, held_until = NULL,
+               created_at = EXCLUDED.created_at
+         WHERE m.held_until < clock_timestamp()
+        RETURNING m.mutex_key
+    `, [holds.map(hold => hold.key), holds.map(hold => hold.id), lockToken])).map(row => String(row.mutex_key)));
+    const withAllKeys = new Set(jobs.filter(job => job.mutexKeys.every(key => acquired.has(key))).map(job => job.id));
+    const partial = holds.filter(hold => !withAllKeys.has(hold.id) && acquired.has(hold.key));
+
+    if (partial.length) {
+        await executor.query(`DELETE FROM ${p}mutexes WHERE mutex_key = ANY($1::text[]) AND lock_token = $2`, [
+            partial.map(hold => hold.key),
+            lockToken,
+        ]);
+    }
+
+    return withAllKeys;
+}
+
+/**
+ * Claims due jobs among the candidates. Call it in a queue transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {ClaimParams} params
+ * @returns {Promise<ClaimedJob[]>} In the order of run_at.
+ */
+export async function claim(executor, p, { candidateIds, lockToken, lockedBy, leaseMs, pick }) {
+    if (!candidateIds.length) {
+        return [];
+    }
+
+    const lockedRows = (await executor.query(`
+        SELECT id, job_name, array_to_json(mutex_keys)::text AS mutex_keys, attempts, max_attempts, stalls,
+               max_stalls, failure_job, ${epochMs('created_at')} AS created_at_ms,
+               (EXTRACT(EPOCH FROM clock_timestamp() - run_at) * 1000)::bigint AS waited_ms
+          FROM ${p}jobs
+         WHERE id = ANY($1::text[]) AND status = 'pending' AND run_at <= clock_timestamp()
+         ORDER BY run_at, id
+           FOR UPDATE SKIP LOCKED
+    `, [candidateIds])).map(row => ({
+        id: String(row.id),
+        name: String(row.job_name),
+        mutexKeys: row.mutex_keys === null ? [] : JSON.parse(String(row.mutex_keys)),
+        attempts: Number(row.attempts),
+        maxAttempts: Number(row.max_attempts),
+        stalls: Number(row.stalls),
+        maxStalls: Number(row.max_stalls),
+        failureJob: row.failure_job === null ? null : String(row.failure_job),
+        enqueuedAt: new Date(Number(row.created_at_ms)),
+        waitedMs: Number(row.waited_ms),
+        lockToken,
+    }));
+    const lockedKeys = [...new Set(lockedRows.flatMap(row => row.mutexKeys))];
+    const busyKeys = lockedKeys.length
+        ? new Set((await executor.query(`
+            SELECT m.mutex_key FROM ${p}mutexes m WHERE m.mutex_key = ANY($1::text[]) AND ${HELD}
+        `, [lockedKeys])).map(row => String(row.mutex_key)))
+        : new Set();
+    const picked = pick(lockedRows, busyKeys);
+    const keyedPicked = picked.filter(row => row.mutexKeys.length);
+    const withKeys = keyedPicked.length ? await takeKeys(executor, p, keyedPicked, lockToken) : new Set();
+    const claimed = picked.filter(row => !row.mutexKeys.length || withKeys.has(row.id));
+
+    if (claimed.length) {
+        await executor.query(`
+            UPDATE ${p}jobs
+               SET status = 'running', lock_token = $2, locked_by = $3,
+                   locked_until = clock_timestamp() + $4 * interval '1 millisecond', started_at = clock_timestamp(),
+                   timed_out_at = NULL, updated_at = clock_timestamp()
+             WHERE id = ANY($1::text[])
+        `, [claimed.map(row => row.id), lockToken, lockedBy, leaseMs]);
+    }
+
+    return claimed;
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} ids
+ * @returns {Promise<Map<string, unknown>>} The payloads by job id, as JSON gives them back.
+ */
+export async function readPayloads(executor, p, ids) {
+    if (!ids.length) {
+        return new Map();
+    }
+
+    const rows = await executor.query(`
+        SELECT id, payload::text AS payload FROM ${p}payloads WHERE id = ANY($1::text[])
+    `, [ids]);
+
+    return new Map(rows.map(row => [String(row.id), JSON.parse(String(row.payload))]));
+}