@sqlane/core 0.1.0 → 0.1.2

package/types/store/mysql_worker.d.ts

@@ -0,0 +1,51 @@
+export type Executor = import('../types.js').Executor;
+export type ProbeRow = import('./store.js').ProbeRow;
+export type Candidate = import('./store.js').Candidate;
+export type ClaimParams = import('./store.js').ClaimParams;
+export type ClaimedJob = import('./store.js').ClaimedJob;
+/** @typedef {import('../types.js').Executor} Executor */
+/** @typedef {import('./store.js').ProbeRow} ProbeRow */
+/** @typedef {import('./store.js').Candidate} Candidate */
+/** @typedef {import('./store.js').ClaimParams} ClaimParams */
+/** @typedef {import('./store.js').ClaimedJob} ClaimedJob */
+/** The due jobs of each keyed name that the FIFO of the mutex keys reads */
+export declare const MUTEX_WINDOW_SIZE = 500;
+/**
+ * The SQL of the probe. `GROUP BY status, job_name`: the whole index prefix, else MySQL does not use a loose index
+ * scan.
+ * @param {string} p
+ * @returns {string}
+ */
+export declare function probeSql(p: string): string;
+/**
+ * The next due time of each job name with pending jobs: one index dive per pending name (loose index scan).
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<ProbeRow[]>}
+ */
+export declare function probe(executor: Executor, p: string): Promise<ProbeRow[]>;
+/**
+ * The due jobs that can start now, oldest first: at most `limit` of each name. A job of a keyed name waits if one of
+ * its keys is held, or an older due job of a keyed name has it (FIFO per key over the window of the keyed names).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {Map<string, number>} limitsByName
+ * @param {string[]} keyedNames The handled names whose definition has mutex keys.
+ * @returns {Promise<Candidate[]>}
+ */
+export declare function findCandidates(executor: Executor, p: string, limitsByName: Map<string, number>, keyedNames: string[]): Promise<Candidate[]>;
+/**
+ * Claims due jobs among the candidates. Call it in a queue transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {ClaimParams} params
+ * @returns {Promise<ClaimedJob[]>} In the order of run_at.
+ */
+export declare function claim(executor: Executor, p: string, { candidateIds, lockToken, lockedBy, leaseMs, pick }: ClaimParams): Promise<ClaimedJob[]>;
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} ids
+ * @returns {Promise<Map<string, unknown>>} The payloads by job id, as JSON gives them back.
+ */
+export declare function readPayloads(executor: Executor, p: string, ids: string[]): Promise<Map<string, unknown>>;

package/types/store/postgres_enqueue.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Inserts the jobs and their payloads.
+ * @param {import('../types.js').Executor} executor
+ * @param {string} p The table prefix.
+ * @param {import('../new_job.js').NewJob[]} jobs Their ids are distinct.
+ * @param {string} enqueueToken
+ * @returns {Promise<Set<string>>} The ids that this call created.
+ */
+export declare function insertJobsPostgres(executor: import('../types.js').Executor, p: string, jobs: import('../new_job.js').NewJob[], enqueueToken: string): Promise<Set<string>>;

package/types/store/postgres_final.d.ts

@@ -0,0 +1,135 @@
+export type Executor = import('../types.js').Executor;
+export type OwnedJob = import('./store.js').OwnedJob;
+export type FailureJobBuilder = import('./store.js').FailureJobBuilder;
+export type FailureJobResult = import('./store.js').FailureJobResult;
+/**
+ * Renews the leases of the jobs that the process runs. A row locked by another transaction is skipped: it does not
+ * block the others.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob[]} jobs
+ * @param {number} leaseMs
+ * @returns {Promise<Set<string>>} The ids whose lease was renewed.
+ */
+export declare function renewLeases(executor: Executor, p: string, jobs: OwnedJob[], leaseMs: number): Promise<Set<string>>;
+/**
+ * Marks a timeout: the job stays running until its handler settles (`stuck` in the stats).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @returns {Promise<boolean>} False: the lease was lost.
+ */
+export declare function markTimedOut(executor: Executor, p: string, job: OwnedJob): Promise<boolean>;
+/**
+ * The jobs are done: completed until their retention ends, without their payloads; their keys are freed. One
+ * statement for many jobs.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {import('./store.js').FinishedJob[]} jobs
+ * @returns {Promise<number>} The jobs that this call completed: the others lost their lease, or an earlier try
+ *     committed.
+ */
+export declare function completeJobs(executor: Executor, p: string, jobs: import('./store.js').FinishedJob[]): Promise<number>;
+/**
+ * The attempt failed: pending again after the delay, one more attempt used; its keys are freed.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, error: import('../error_json.js').ErrorJson }} params
+ * @returns {Promise<boolean>} False: no change.
+ */
+export declare function retryJob(executor: Executor, p: string, job: OwnedJob, { delayMs, error }: {
+    delayMs: number;
+    error: import('../error_json.js').ErrorJson;
+}): Promise<boolean>;
+/**
+ * The job fails for good: failed until its retention ends, with its payload; its keys are freed; its failure job is
+ * added in the same transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ changed: boolean, failureJob: FailureJobResult | null }>}
+ */
+export declare function failJob(executor: Executor, p: string, job: OwnedJob, { error, buildFailureJob }: {
+    error: import('../error_json.js').ErrorJson;
+    buildFailureJob: FailureJobBuilder;
+}): Promise<{
+    changed: boolean;
+    failureJob: FailureJobResult | null;
+}>;
+/**
+ * The job is released, without an attempt used: pending again after the delay.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {OwnedJob} job
+ * @param {{ delayMs: number, lingerMs: number }} params lingerMs: its keys stay held this long (its handler still
+ *     runs); 0: they are freed.
+ * @returns {Promise<boolean>} False: no change.
+ */
+export declare function releaseJob(executor: Executor, p: string, job: OwnedJob, { delayMs, lingerMs }: {
+    delayMs: number;
+    lingerMs: number;
+}): Promise<boolean>;
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ id: string, name: string }[]>} The running jobs whose lease ended, the oldest first.
+ */
+export declare function findStaleJobs(executor: Executor, p: string, limit: number): Promise<{
+    id: string;
+    name: string;
+}[]>;
+/**
+ * A running job whose lease ended: one more stall, pending again after 30 s × 2^stalls (at most 1 h); after maxStalls,
+ * it fails for good. Its keys linger for one lease: its old handler may still run.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string} id
+ * @param {{ leaseMs: number, buildFailureJob: FailureJobBuilder }} params
+ * @returns {Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} null: not stale
+ *     (any more).
+ */
+export declare function recoverStaleJob(executor: Executor, p: string, id: string, { leaseMs, buildFailureJob }: {
+    leaseMs: number;
+    buildFailureJob: FailureJobBuilder;
+}): Promise<{
+    result: 'released' | 'failed' | null;
+    failureJob: FailureJobResult | null;
+}>;
+/**
+ * Deletes the ended lingers, and the holds without their running job (from a manual change, or old code).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<{ lingers: number, orphans: number }>}
+ */
+export declare function sweepMutexes(executor: Executor, p: string, limit: number): Promise<{
+    lingers: number;
+    orphans: number;
+}>;
+/**
+ * Deletes a batch of completed and failed jobs whose retention ended, with their payloads. SKIP LOCKED: processes
+ * share the work.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} limit
+ * @returns {Promise<number>} The deleted jobs.
+ */
+export declare function deleteExpiredJobs(executor: Executor, p: string, limit: number): Promise<number>;
+/**
+ * For drain({ includeDelayed }): the pending jobs of the names are due now.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} names
+ * @returns {Promise<void>}
+ */
+export declare function makeDue(executor: Executor, p: string, names: string[]): Promise<void>;
+/**
+ * The counts per job name, for alerts such as "queue stuck".
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<import('../types.js').JobStats[]>}
+ */
+export declare function getStats(executor: Executor, p: string): Promise<import('../types.js').JobStats[]>;

package/types/store/postgres_worker.d.ts

@@ -0,0 +1,45 @@
+export type Executor = import('../types.js').Executor;
+export type ProbeRow = import('./store.js').ProbeRow;
+export type Candidate = import('./store.js').Candidate;
+export type ClaimParams = import('./store.js').ClaimParams;
+export type ClaimedJob = import('./store.js').ClaimedJob;
+/** @typedef {import('../types.js').Executor} Executor */
+/** @typedef {import('./store.js').ProbeRow} ProbeRow */
+/** @typedef {import('./store.js').Candidate} Candidate */
+/** @typedef {import('./store.js').ClaimParams} ClaimParams */
+/** @typedef {import('./store.js').ClaimedJob} ClaimedJob */
+/** The due jobs of each keyed name that the FIFO of the mutex keys reads */
+export declare const MUTEX_WINDOW_SIZE = 500;
+/**
+ * The next due time of each job name with pending jobs. PostgreSQL has no loose index scan: a recursive CTE skips from
+ * one pending name to the next on the partial index, two index probes per pending name.
+ * @param {Executor} executor
+ * @param {string} p
+ * @returns {Promise<ProbeRow[]>}
+ */
+export declare function probe(executor: Executor, p: string): Promise<ProbeRow[]>;
+/**
+ * The due jobs that can start now, oldest first: at most `limit` of each name. A job of a keyed name waits if one of
+ * its keys is held, or an older due job of a keyed name has it (FIFO per key over the window of the keyed names).
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {Map<string, number>} limitsByName
+ * @param {string[]} keyedNames The handled names whose definition has mutex keys.
+ * @returns {Promise<Candidate[]>}
+ */
+export declare function findCandidates(executor: Executor, p: string, limitsByName: Map<string, number>, keyedNames: string[]): Promise<Candidate[]>;
+/**
+ * Claims due jobs among the candidates. Call it in a queue transaction.
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {ClaimParams} params
+ * @returns {Promise<ClaimedJob[]>} In the order of run_at.
+ */
+export declare function claim(executor: Executor, p: string, { candidateIds, lockToken, lockedBy, leaseMs, pick }: ClaimParams): Promise<ClaimedJob[]>;
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {string[]} ids
+ * @returns {Promise<Map<string, unknown>>} The payloads by job id, as JSON gives them back.
+ */
+export declare function readPayloads(executor: Executor, p: string, ids: string[]): Promise<Map<string, unknown>>;

package/types/store/store.d.ts

@@ -0,0 +1,221 @@
+export type ProbeRow = {
+    /**
+     * A job name with pending jobs.
+     */
+    name: string;
+    /**
+     * Until its oldest pending job is due: 0 or less means due now.
+     */
+    delayMs: number;
+};
+export type Candidate = {
+    id: string;
+    name: string;
+    runAtMs: number;
+};
+export type ClaimedJob = {
+    id: string;
+    name: string;
+    mutexKeys: string[];
+    /**
+     * The failed attempts so far.
+     */
+    attempts: number;
+    maxAttempts: number;
+    stalls: number;
+    maxStalls: number;
+    failureJob: string | null;
+    enqueuedAt: Date;
+    /**
+     * Since it was due.
+     */
+    waitedMs: number;
+    lockToken: string;
+};
+export type ClaimParams = {
+    candidateIds: string[];
+    /**
+     * Of this claim.
+     */
+    lockToken: string;
+    /**
+     * The process.
+     */
+    lockedBy: string;
+    leaseMs: number;
+    /**
+     * The jobs to take among the locked
+     * ones (in the order of run_at): the free slots, the free keys. The keys of the picked jobs must be distinct.
+     */
+    pick: (locked: ClaimedJob[], busyKeys: Set<string>) => ClaimedJob[];
+};
+export type OwnedJob = {
+    id: string;
+    lockToken: string;
+};
+export type FinishedJob = OwnedJob & {
+    mutexKeys: readonly string[];
+};
+export type FailedJob = {
+    id: string;
+    name: string;
+    /**
+     * Null: its payload is missing.
+     */
+    payload: unknown;
+    attempt: number;
+    maxAttempts: number;
+    /**
+     * The name of its failure job.
+     */
+    failureJob: string;
+    error: import('../error_json.js').ErrorJson;
+};
+export type FailureJobBuilder = (failed: FailedJob) => import('../new_job.js').NewJob;
+export type FailureJobResult = {
+    added: import('../new_job.js').NewJob;
+} | {
+    notAdded: unknown;
+};
+export type Store = {
+    probe: () => Promise<ProbeRow[]>;
+    findCandidates: (limitsByName: Map<string, number>, keyedNames: string[]) => Promise<Candidate[]>;
+    /**
+     * In a queue transaction of its own.
+     */
+    claim: (params: ClaimParams) => Promise<ClaimedJob[]>;
+    readPayloads: (ids: string[]) => Promise<Map<string, unknown>>;
+    renewLeases: (jobs: OwnedJob[], leaseMs: number) => Promise<Set<string>>;
+    markTimedOut: (job: OwnedJob) => Promise<boolean>;
+    /**
+     * In one transaction: the jobs that it completed.
+     */
+    completeJobs: (jobs: FinishedJob[]) => Promise<number>;
+    retryJob: (job: OwnedJob, params: {
+        delayMs: number;
+        error: import('../error_json.js').ErrorJson;
+    }) => Promise<boolean>;
+    failJob: (job: OwnedJob, params: {
+        error: import('../error_json.js').ErrorJson;
+        buildFailureJob: FailureJobBuilder;
+    }) => Promise<{
+        changed: boolean;
+        failureJob: FailureJobResult | null;
+    }>;
+    releaseJob: (job: OwnedJob, params: {
+        delayMs: number;
+        lingerMs: number;
+    }) => Promise<boolean>;
+    findStaleJobs: (limit: number) => Promise<{
+        id: string;
+        name: string;
+    }[]>;
+    recoverStaleJob: (id: string, params: {
+        leaseMs: number;
+        buildFailureJob: FailureJobBuilder;
+    }) => Promise<{
+        result: 'released' | 'failed' | null;
+        failureJob: FailureJobResult | null;
+    }>;
+    sweepMutexes: (limit: number) => Promise<{
+        lingers: number;
+        orphans: number;
+    }>;
+    deleteExpiredJobs: (limit: number) => Promise<number>;
+    /**
+     * For drain({ includeDelayed }).
+     */
+    makeDue: (names: string[]) => Promise<void>;
+    getStats: () => Promise<import('../types.js').JobStats[]>;
+};
+/**
+ * @typedef {object} ProbeRow
+ * @property {string} name A job name with pending jobs.
+ * @property {number} delayMs Until its oldest pending job is due: 0 or less means due now.
+ */
+/**
+ * @typedef {object} Candidate
+ * @property {string} id
+ * @property {string} name
+ * @property {number} runAtMs
+ */
+/**
+ * A job that a claim locked, and then claimed: it runs.
+ * @typedef {object} ClaimedJob
+ * @property {string} id
+ * @property {string} name
+ * @property {string[]} mutexKeys
+ * @property {number} attempts The failed attempts so far.
+ * @property {number} maxAttempts
+ * @property {number} stalls
+ * @property {number} maxStalls
+ * @property {string | null} failureJob
+ * @property {Date} enqueuedAt
+ * @property {number} waitedMs Since it was due.
+ * @property {string} lockToken
+ */
+/**
+ * @typedef {object} ClaimParams
+ * @property {string[]} candidateIds
+ * @property {string} lockToken Of this claim.
+ * @property {string} lockedBy The process.
+ * @property {number} leaseMs
+ * @property {(locked: ClaimedJob[], busyKeys: Set<string>) => ClaimedJob[]} pick The jobs to take among the locked
+ *     ones (in the order of run_at): the free slots, the free keys. The keys of the picked jobs must be distinct.
+ */
+/**
+ * A job that this process claimed: every write of it checks the lock token.
+ * @typedef {object} OwnedJob
+ * @property {string} id
+ * @property {string} lockToken
+ */
+/**
+ * A job that this process ran, with its keys.
+ * @typedef {OwnedJob & { mutexKeys: readonly string[] }} FinishedJob
+ */
+/**
+ * The job that fails for good, as the failure job gets it.
+ * @typedef {object} FailedJob
+ * @property {string} id
+ * @property {string} name
+ * @property {unknown} payload Null: its payload is missing.
+ * @property {number} attempt
+ * @property {number} maxAttempts
+ * @property {string} failureJob The name of its failure job.
+ * @property {import('../error_json.js').ErrorJson} error
+ */
+/**
+ * Builds the failure job: its options from its definition, else the defaults; no failure job of its own.
+ * @typedef {(failed: FailedJob) => import('../new_job.js').NewJob} FailureJobBuilder
+ */
+/** @typedef {{ added: import('../new_job.js').NewJob } | { notAdded: unknown }} FailureJobResult */
+/**
+ * @typedef {object} Store
+ * @property {() => Promise<ProbeRow[]>} probe
+ * @property {(limitsByName: Map<string, number>, keyedNames: string[]) => Promise<Candidate[]>} findCandidates
+ * @property {(params: ClaimParams) => Promise<ClaimedJob[]>} claim In a queue transaction of its own.
+ * @property {(ids: string[]) => Promise<Map<string, unknown>>} readPayloads
+ * @property {(jobs: OwnedJob[], leaseMs: number) => Promise<Set<string>>} renewLeases
+ * @property {(job: OwnedJob) => Promise<boolean>} markTimedOut
+ * @property {(jobs: FinishedJob[]) => Promise<number>} completeJobs In one transaction: the jobs that it completed.
+ * @property {(job: OwnedJob, params: { delayMs: number, error: import('../error_json.js').ErrorJson })
+ *     => Promise<boolean>} retryJob
+ * @property {(
+ *     job: OwnedJob,
+ *     params: { error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder },
+ * ) => Promise<{ changed: boolean, failureJob: FailureJobResult | null }>} failJob
+ * @property {(job: OwnedJob, params: { delayMs: number, lingerMs: number }) => Promise<boolean>} releaseJob
+ * @property {(limit: number) => Promise<{ id: string, name: string }[]>} findStaleJobs
+ * @property {(id: string, params: { leaseMs: number, buildFailureJob: FailureJobBuilder })
+ *     => Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} recoverStaleJob
+ * @property {(limit: number) => Promise<{ lingers: number, orphans: number }>} sweepMutexes
+ * @property {(limit: number) => Promise<number>} deleteExpiredJobs
+ * @property {(names: string[]) => Promise<void>} makeDue For drain({ includeDelayed }).
+ * @property {() => Promise<import('../types.js').JobStats[]>} getStats
+ */
+/**
+ * @param {import('../types.js').Adapter<any>} adapter
+ * @param {string} tablePrefix Checked.
+ * @returns {Store}
+ */
+export declare function createStore(adapter: import('../types.js').Adapter<any>, tablePrefix: string): Store;

package/types/transient_error.d.ts

@@ -0,0 +1,6 @@
+/**
+ * A database error that a retry of the same work can pass: the transaction lost a lock race.
+ * @param {unknown} error
+ * @returns {boolean}
+ */
+export declare function isTransientDbError(error: unknown): boolean;