@sqlane/core 0.0.2 → 0.1.0

package/src/store/store.js

@@ -0,0 +1,136 @@
+// The store: the SQL of one dialect, bound to an adapter and a table prefix.
+import * as mysqlFinal from './mysql_final.js';
+import * as mysqlWorker from './mysql_worker.js';
+import * as postgresFinal from './postgres_final.js';
+import * as postgresWorker from './postgres_worker.js';
+
+/**
+ * @typedef {object} ProbeRow
+ * @property {string} name A job name with pending jobs.
+ * @property {number} delayMs Until its oldest pending job is due: 0 or less means due now.
+ */
+
+/**
+ * @typedef {object} Candidate
+ * @property {string} id
+ * @property {string} name
+ * @property {number} runAtMs
+ */
+
+/**
+ * A job that a claim locked, and then claimed: it runs.
+ * @typedef {object} ClaimedJob
+ * @property {string} id
+ * @property {string} name
+ * @property {string[]} mutexKeys
+ * @property {number} attempts The failed attempts so far.
+ * @property {number} maxAttempts
+ * @property {number} stalls
+ * @property {number} maxStalls
+ * @property {string | null} failureJob
+ * @property {Date} enqueuedAt
+ * @property {number} waitedMs Since it was due.
+ * @property {string} lockToken
+ */
+
+/**
+ * @typedef {object} ClaimParams
+ * @property {string[]} candidateIds
+ * @property {string} lockToken Of this claim.
+ * @property {string} lockedBy The process.
+ * @property {number} leaseMs
+ * @property {(locked: ClaimedJob[], busyKeys: Set<string>) => ClaimedJob[]} pick The jobs to take among the locked
+ *     ones (in the order of run_at): the free slots, the free keys. The keys of the picked jobs must be distinct.
+ */
+
+/**
+ * A job that this process claimed: every write of it checks the lock token.
+ * @typedef {object} OwnedJob
+ * @property {string} id
+ * @property {string} lockToken
+ */
+
+/**
+ * The job that fails for good, as the failure job gets it.
+ * @typedef {object} FailedJob
+ * @property {string} id
+ * @property {string} name
+ * @property {unknown} payload Null: its payload is missing.
+ * @property {number} attempt
+ * @property {number} maxAttempts
+ * @property {string} failureJob The name of its failure job.
+ * @property {import('../error_json.js').ErrorJson} error
+ */
+
+/**
+ * Builds the failure job: its options from its definition, else the defaults; no failure job of its own.
+ * @typedef {(failed: FailedJob) => import('../new_job.js').NewJob} FailureJobBuilder
+ */
+
+/** @typedef {{ added: import('../new_job.js').NewJob } | { notAdded: unknown }} FailureJobResult */
+
+/**
+ * @typedef {object} Store
+ * @property {() => Promise<ProbeRow[]>} probe
+ * @property {(limitsByName: Map<string, number>, keyedNames: string[]) => Promise<Candidate[]>} findCandidates
+ * @property {(params: ClaimParams) => Promise<ClaimedJob[]>} claim In a queue transaction of its own.
+ * @property {(ids: string[]) => Promise<Map<string, unknown>>} readPayloads
+ * @property {(jobs: OwnedJob[], leaseMs: number) => Promise<Set<string>>} renewLeases
+ * @property {(job: OwnedJob) => Promise<boolean>} markTimedOut
+ * @property {(job: OwnedJob) => Promise<boolean>} completeJob
+ * @property {(job: OwnedJob, params: { delayMs: number, error: import('../error_json.js').ErrorJson })
+ *     => Promise<boolean>} retryJob
+ * @property {(
+ *     job: OwnedJob,
+ *     params: { error: import('../error_json.js').ErrorJson, buildFailureJob: FailureJobBuilder },
+ * ) => Promise<{ changed: boolean, failureJob: FailureJobResult | null }>} failJob
+ * @property {(job: OwnedJob, params: { delayMs: number, lingerMs: number }) => Promise<boolean>} releaseJob
+ * @property {(limit: number) => Promise<{ id: string, name: string }[]>} findStaleJobs
+ * @property {(id: string, params: { leaseMs: number, buildFailureJob: FailureJobBuilder })
+ *     => Promise<{ result: 'released' | 'failed' | null, failureJob: FailureJobResult | null }>} recoverStaleJob
+ * @property {(limit: number) => Promise<{ lingers: number, orphans: number }>} sweepMutexes
+ * @property {(limit: number) => Promise<number>} deleteExpiredJobs
+ * @property {(names: string[]) => Promise<void>} makeDue For drain({ includeDelayed }).
+ * @property {() => Promise<import('../types.js').JobStats[]>} getStats
+ */
+
+/**
+ * @param {import('../types.js').Adapter<any>} adapter
+ * @param {string} tablePrefix Checked.
+ * @returns {Store}
+ */
+export function createStore(adapter, tablePrefix) {
+    const sql = adapter.dialect === 'mysql' ? mysqlWorker : postgresWorker;
+    const final = adapter.dialect === 'mysql' ? mysqlFinal : postgresFinal;
+    /**
+     * @template T
+     * @param {(executor: import('../types.js').Executor) => Promise<T>} func
+     * @returns {Promise<T>}
+     */
+    const inQueue = func => adapter.queueTransaction(func);
+    const p = tablePrefix;
+
+    return {
+        probe: () => sql.probe(adapter.queue, tablePrefix),
+        findCandidates: (limitsByName, keyedNames) => sql.findCandidates(
+            adapter.queue,
+            tablePrefix,
+            limitsByName,
+            keyedNames,
+        ),
+        claim: params => adapter.queueTransaction(executor => sql.claim(executor, tablePrefix, params)),
+        readPayloads: ids => sql.readPayloads(adapter.queue, tablePrefix, ids),
+        renewLeases: (jobs, leaseMs) => inQueue(executor => final.renewLeases(executor, p, jobs, leaseMs)),
+        markTimedOut: job => final.markTimedOut(adapter.queue, p, job),
+        completeJob: job => inQueue(executor => final.completeJob(executor, p, job)),
+        retryJob: (job, params) => inQueue(executor => final.retryJob(executor, p, job, params)),
+        failJob: (job, params) => inQueue(executor => final.failJob(executor, p, job, params)),
+        releaseJob: (job, params) => inQueue(executor => final.releaseJob(executor, p, job, params)),
+        findStaleJobs: limit => final.findStaleJobs(adapter.queue, p, limit),
+        recoverStaleJob: (id, params) => inQueue(executor => final.recoverStaleJob(executor, p, id, params)),
+        sweepMutexes: limit => inQueue(executor => final.sweepMutexes(executor, p, limit)),
+        deleteExpiredJobs: limit => inQueue(executor => final.deleteExpiredJobs(executor, p, limit)),
+        makeDue: names => inQueue(executor => final.makeDue(executor, p, names)),
+        getStats: () => final.getStats(adapter.queue, p),
+    };
+}

package/src/transient_error.js

@@ -0,0 +1,22 @@
+/** MySQL: deadlock, lock wait timeout. PostgreSQL: serialization failure, deadlock. */
+const TRANSIENT_CODES = new Set(['ER_LOCK_DEADLOCK', 'ER_LOCK_WAIT_TIMEOUT', '40001', '40P01']);
+const TRANSIENT_ERRNOS = new Set([1205, 1213]);
+
+/**
+ * A database error that a retry of the same work can pass: the transaction lost a lock race.
+ * @param {unknown} error
+ * @returns {boolean}
+ */
+export function isTransientDbError(error) {
+    for (let current = error, depth = 0; current && depth < 5; depth++) {
+        const { code, errno, cause } = /** @type {{ code?: unknown, errno?: unknown, cause?: unknown }} */ (current);
+
+        if (TRANSIENT_CODES.has(String(code)) || TRANSIENT_ERRNOS.has(Number(errno))) {
+            return true;
+        }
+
+        current = cause;
+    }
+
+    return false;
+}

package/src/types.js

@@ -1,7 +1,7 @@
 // The public types of sqlane. See docs/design.md.
 
 /**
- * A duration: a string (`'500ms'`, `'5s'`, `'10m'`, `'1h'`, `'30d'`) or milliseconds.
+ * A duration: milliseconds, or a string of parse-duration (`'500ms'`, `'5s'`, `'4m30s'`, `'1h'`, `'2 days'`).
  * @typedef {string | number} Duration
  */
 
@@ -183,12 +183,23 @@
  * @property {'mysql' | 'postgres'} dialect
  * @property {(connection: C) => Executor} wrap Used by sqlane: wraps a connection given to `enqueue`.
  * @property {Executor} queue Used by sqlane: the queue's own pool.
+ * @property {<T>(func: (executor: Executor) => Promise<T>) => Promise<T>} withConnection Used by sqlane: runs func on
+ *     one connection of the queue's pool, then releases it.
+ * @property {<T>(func: (executor: Executor) => Promise<T>) => Promise<T>} queueTransaction Used by sqlane: runs func
+ *     in a `READ COMMITTED` transaction on a connection of the queue's pool.
+ * @property {<T>(connection: C, func: (executor: Executor) => Promise<T>) => Promise<T>} inTransaction Used by sqlane:
+ *     a pool → func in a transaction of its own; a connection or a client → func on it, in the caller's transaction.
+ * @property {boolean} [notify] PostgreSQL: hints by LISTEN/NOTIFY.
+ * @property {(channel: string, onMessage: (message: string) => void) => Promise<() => Promise<void>>} [listen] Used by
+ *     sqlane, with `notify`: LISTEN on a connection of its own; resolves to the unlisten function.
  */
 
 /**
  * Runs SQL. Used by sqlane, through an adapter.
  * @typedef {object} Executor
  * @property {(sql: string, params: readonly unknown[]) => Promise<readonly Record<string, unknown>[]>} query
+ * @property {(sql: string, params: readonly unknown[]) => Promise<number>} run Runs an INSERT, UPDATE or DELETE:
+ *     the number of rows it changed.
  */
 
 /**
@@ -273,7 +284,8 @@
  * @property {(options?: DrainOptions) => Promise<DrainResult>} drain For tests: runs the due jobs of the handled
  *     names until none is left, without `start()`.
  * @property {(options?: CloseOptions) => Promise<void>} close
- * @property {(options?: MigrateOptions) => Promise<void>} migrate
+ * @property {(options?: MigrateOptions) => Promise<{ applied: string[] }>} migrate Applies the missing migrations:
+ *     their ids.
  * @property {() => Promise<JobStats[]>} getStats
  * @property {(jobName: string) => void} wake Low level: probes this name now.
  */

package/src/validate.js

@@ -0,0 +1,181 @@
+import { parseDuration } from './duration.js';
+
+/** A job name: no `:` (the job id is `<name>:<jobId>`), lowercase. */
+export const JOB_NAME_REGEXP = /^[a-z][a-z0-9_.-]{0,127}$/;
+
+/** A table prefix: the longest table name, `<prefix>migrations`, stays within the 63 characters of PostgreSQL. */
+export const TABLE_PREFIX_REGEXP = /^[a-z][a-z0-9_]{0,31}$/;
+
+/** Printable ASCII, no space at the ends: MySQL `ascii_bin` ignores trailing spaces in comparisons. */
+export const PRINTABLE_ID_REGEXP = /^[\x21-\x7E](?:[\x20-\x7E]*[\x21-\x7E])?$/;
+
+/**
+ * Where a check runs: for the error message and its class.
+ * @typedef {object} Scope
+ * @property {string} where E.g. `defineJob("build_report")`.
+ * @property {new (message: string) => Error} ErrorClass
+ */
+
+/**
+ * @param {unknown} value
+ * @returns {value is Record<string, unknown>}
+ */
+export function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Throws if the value is not an object with only the allowed keys.
+ * @param {unknown} value
+ * @param {readonly string[]} allowedKeys
+ * @param {string} label E.g. `"options"`.
+ * @param {Scope} scope
+ * @returns {Record<string, unknown>}
+ */
+export function checkObject(value, allowedKeys, label, { where, ErrorClass }) {
+    if (!isPlainObject(value)) {
+        throw new ErrorClass(`${where}: ${label} must be an object`);
+    }
+
+    const unknownKeys = Object.keys(value).filter(key => !allowedKeys.includes(key));
+
+    if (unknownKeys.length) {
+        throw new ErrorClass(`${where}: unknown ${label}: ${unknownKeys.join(', ')}`);
+    }
+
+    return value;
+}
+
+/**
+ * @param {unknown} value
+ * @param {string} label
+ * @param {Scope} scope
+ * @param {number} [min] Default: 1.
+ */
+export function checkInteger(value, label, { where, ErrorClass }, min = 1) {
+    if (!Number.isSafeInteger(value) || /** @type {number} */ (value) < min) {
+        throw new ErrorClass(`${where}: ${label} must be an integer of at least ${min}, got ${JSON.stringify(value)}`);
+    }
+}
+
+/**
+ * @param {unknown} value
+ * @param {string} label
+ * @param {Scope} scope
+ * @param {boolean} [positive] Default: false: 0 is a duration.
+ * @returns {number} Milliseconds.
+ */
+export function checkDuration(value, label, { where, ErrorClass }, positive = false) {
+    const ms = parseDuration(value);
+
+    if (ms === null || (positive && ms === 0)) {
+        throw new ErrorClass(
+            `${where}: ${label} must be a ${positive ? 'positive ' : ''}duration ('5s', '4m30s', or milliseconds), `
+            + `got ${JSON.stringify(value)}`,
+        );
+    }
+
+    return ms;
+}
+
+/**
+ * @param {unknown} value
+ * @param {Scope} scope
+ */
+export function checkBackoff(value, scope) {
+    const { where, ErrorClass } = scope;
+    const backoff = checkObject(value, ['type', 'delay', 'maxDelay', 'jitter'], '"backoff" options', scope);
+
+    if (backoff.type !== 'exponential' && backoff.type !== 'fixed') {
+        throw new ErrorClass(`${where}: "backoff.type" must be 'exponential' or 'fixed'`);
+    }
+
+    const delayMs = checkDuration(backoff.delay, '"backoff.delay"', scope);
+
+    if (backoff.maxDelay !== undefined) {
+        if (backoff.type === 'fixed') {
+            throw new ErrorClass(`${where}: "backoff.maxDelay" is only for the exponential backoff`);
+        }
+
+        if (checkDuration(backoff.maxDelay, '"backoff.maxDelay"', scope) < delayMs) {
+            throw new ErrorClass(`${where}: "backoff.maxDelay" must be at least "backoff.delay"`);
+        }
+    }
+
+    const { jitter } = backoff;
+
+    if (jitter !== undefined && (typeof jitter !== 'number' || !(jitter >= 0 && jitter <= 1))) {
+        throw new ErrorClass(`${where}: "backoff.jitter" must be a number from 0 to 1`);
+    }
+}
+
+/**
+ * @param {unknown} value
+ * @param {Scope} scope
+ */
+export function checkRetention(value, scope) {
+    const retention = checkObject(value, ['completed', 'failed'], '"retention" options', scope);
+
+    for (const key of ['completed', 'failed']) {
+        if (retention[key] !== undefined) {
+            checkDuration(retention[key], `"retention.${key}"`, scope);
+        }
+    }
+}
+
+/**
+ * Checks the options that a definition, the instance defaults and an enqueue share.
+ * @param {Record<string, unknown>} options
+ * @param {Scope} scope
+ */
+export function checkJobFields(options, scope) {
+    if (options.maxAttempts !== undefined) {
+        checkInteger(options.maxAttempts, '"maxAttempts"', scope);
+    }
+
+    if (options.maxStalls !== undefined) {
+        checkInteger(options.maxStalls, '"maxStalls"', scope, 0);
+    }
+
+    if (options.backoff !== undefined) {
+        checkBackoff(options.backoff, scope);
+    }
+
+    if (options.retention !== undefined) {
+        checkRetention(options.retention, scope);
+    }
+
+    if (options.timeout !== undefined) {
+        checkDuration(options.timeout, '"timeout"', scope, true);
+    }
+}
+
+/**
+ * @param {unknown} value A name, or a definition.
+ * @param {string} jobName The job that names it.
+ * @param {Scope} scope
+ * @returns {string} The name of the failure job.
+ */
+export function checkFailureJob(value, jobName, { where, ErrorClass }) {
+    const name = isPlainObject(value) ? value.name : value;
+
+    if (typeof name !== 'string' || !JOB_NAME_REGEXP.test(name)) {
+        throw new ErrorClass(`${where}: "failureJob" must be a job name or a job definition`);
+    }
+
+    if (name === jobName) {
+        throw new ErrorClass(`${where}: a job cannot be its own failure job`);
+    }
+
+    return name;
+}
+
+/**
+ * @param {unknown} tablePrefix
+ * @param {Scope} scope
+ */
+export function checkTablePrefix(tablePrefix, { where, ErrorClass }) {
+    if (typeof tablePrefix !== 'string' || !TABLE_PREFIX_REGEXP.test(tablePrefix)) {
+        throw new ErrorClass(`${where}: "tablePrefix" must match ${TABLE_PREFIX_REGEXP}`);
+    }
+}