@sqlane/core 0.0.1 → 0.1.0

package/src/migrations.js

@@ -0,0 +1,163 @@
+import { parseDuration } from './duration.js';
+import { ConfigError, InvalidOptionsError, MigrationLockTimeoutError } from './errors.js';
+import { mysqlMigrations, mysqlMigrationsTable } from './schema/mysql.js';
+import { postgresMigrations, postgresMigrationsTable } from './schema/postgres.js';
+import { checkTablePrefix } from './validate.js';
+
+/** @typedef {import('./types.js').Executor} Executor */
+
+/**
+ * The schema of sqlane as versioned SQL, for the app's own migration tool.
+ * @param {object} options
+ * @param {'mysql' | 'postgres'} options.dialect
+ * @param {string} [options.tablePrefix] Default: `'sqlane_'`.
+ * @returns {import('./types.js').Migration[]}
+ */
+export function getMigrations({ dialect, tablePrefix = 'sqlane_' }) {
+    const scope = { where: 'getMigrations', ErrorClass: ConfigError };
+
+    checkTablePrefix(tablePrefix, scope);
+
+    if (dialect === 'mysql') {
+        return mysqlMigrations(tablePrefix);
+    }
+
+    if (dialect === 'postgres') {
+        return postgresMigrations(tablePrefix);
+    }
+
+    throw new ConfigError(`getMigrations: "dialect" must be 'mysql' or 'postgres'`);
+}
+
+/**
+ * Runs the statements one after the other: DDL in order.
+ * @param {Executor} executor
+ * @param {{ sql: string, params: unknown[] }[]} steps
+ * @returns {Promise<void>}
+ */
+async function runInOrder(executor, steps) {
+    for (const { sql, params } of steps) {
+        // oxlint-disable-next-line no-await-in-loop -- each statement needs the previous one
+        await executor.query(sql, params);
+    }
+}
+
+/**
+ * The statements of the missing migrations, each followed by its record.
+ * @param {import('./types.js').Migration[]} missing
+ * @param {string} recordSql
+ * @returns {{ sql: string, params: unknown[] }[]}
+ */
+function toSteps(missing, recordSql) {
+    return missing.flatMap(({ id, statements }) => [
+        ...statements.map(sql => ({ sql, params: [] })),
+        { sql: recordSql, params: [id] },
+    ]);
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} table
+ * @returns {Promise<Set<string>>}
+ */
+async function readApplied(executor, table) {
+    const rows = await executor.query(`SELECT id FROM ${table}`, []);
+
+    return new Set(rows.map(row => String(row.id)));
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} lockTimeoutMs
+ * @returns {Promise<string[]>}
+ */
+async function migrateMysql(executor, p, lockTimeoutMs) {
+    const lockName = `${p}migrations`;
+    // GET_LOCK waits whole seconds; -1 is forever, so at least 1
+    const [{ locked }] = await executor.query('SELECT GET_LOCK(?, ?) AS locked', [
+        lockName,
+        Math.max(1, Math.ceil(lockTimeoutMs / 1000)),
+    ]);
+
+    if (Number(locked) !== 1) {
+        throw new MigrationLockTimeoutError(`migrate: the lock "${lockName}" was not free in ${lockTimeoutMs} ms`);
+    }
+
+    try {
+        await executor.query(mysqlMigrationsTable(p), []);
+
+        const applied = await readApplied(executor, `${p}migrations`);
+        const missing = mysqlMigrations(p).filter(({ id }) => !applied.has(id));
+
+        await runInOrder(executor, toSteps(missing, `INSERT INTO ${p}migrations (id, applied_at) VALUES (?, UTC_TIMESTAMP(3))`));
+
+        return missing.map(({ id }) => id);
+    } finally {
+        await executor.query('SELECT RELEASE_LOCK(?)', [lockName]);
+    }
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} lockTimeoutMs
+ * @returns {Promise<string[]>}
+ */
+async function migratePostgres(executor, p, lockTimeoutMs) {
+    await executor.query('BEGIN', []);
+
+    try {
+        // SET LOCAL: until the end of the transaction, then the pooled connection gets its settings back
+        await executor.query('SET LOCAL statement_timeout = 0', []);
+        await executor.query(`SET LOCAL lock_timeout = ${Math.max(1, lockTimeoutMs)}`, []);
+
+        try {
+            await executor.query('SELECT pg_advisory_xact_lock(hashtext($1))', [`${p}migrations`]);
+        } catch (error) {
+            if (/** @type {{ code?: string }} */ (error).code === '55P03') {
+                throw new MigrationLockTimeoutError(
+                    `migrate: the lock of "${p}migrations" was not free in ${lockTimeoutMs} ms`,
+                    { cause: error },
+                );
+            }
+
+            throw error;
+        }
+
+        await executor.query('SET LOCAL lock_timeout = 0', []);
+        await executor.query(postgresMigrationsTable(p), []);
+
+        const applied = await readApplied(executor, `${p}migrations`);
+        const missing = postgresMigrations(p).filter(({ id }) => !applied.has(id));
+
+        await runInOrder(executor, toSteps(missing, `INSERT INTO ${p}migrations (id, applied_at) VALUES ($1, clock_timestamp())`));
+
+        await executor.query('COMMIT', []);
+
+        return missing.map(({ id }) => id);
+    } catch (error) {
+        await executor.query('ROLLBACK', []);
+        throw error;
+    }
+}
+
+/**
+ * Applies the missing migrations under a DB lock, on a connection of its own, without a statement timeout.
+ * @template C
+ * @param {import('./types.js').Adapter<C>} adapter
+ * @param {string} tablePrefix Checked.
+ * @param {import('./types.js').MigrateOptions} [options]
+ * @returns {Promise<string[]>} The ids of the applied migrations.
+ */
+export function runMigrations(adapter, tablePrefix, options = {}) {
+    const lockTimeoutMs = parseDuration(options.lockTimeout ?? '10m');
+
+    if (lockTimeoutMs === null) {
+        return Promise.reject(new InvalidOptionsError('migrate: "lockTimeout" must be a duration'));
+    }
+
+    return adapter.withConnection(executor => (adapter.dialect === 'mysql'
+        ? migrateMysql(executor, tablePrefix, lockTimeoutMs)
+        : migratePostgres(executor, tablePrefix, lockTimeoutMs)));
+}

package/src/mutex_keys.js

@@ -0,0 +1,34 @@
+import { InvalidMutexKeyError } from './errors.js';
+import { PRINTABLE_ID_REGEXP } from './validate.js';
+
+export const MAX_MUTEX_KEYS = 10;
+export const MAX_MUTEX_KEY_LENGTH = 255;
+
+/**
+ * Checks the mutex keys of a job.
+ * @param {unknown} keys
+ * @param {string} where For the error message.
+ * @returns {string[]} The keys without duplicates, sorted (the lock order).
+ */
+export function normalizeMutexKeys(keys, where) {
+    if (!Array.isArray(keys)) {
+        throw new InvalidMutexKeyError(`${where}: the mutex keys must be an array`);
+    }
+
+    for (const key of keys) {
+        if (typeof key !== 'string' || key.length > MAX_MUTEX_KEY_LENGTH || !PRINTABLE_ID_REGEXP.test(key)) {
+            throw new InvalidMutexKeyError(
+                `${where}: a mutex key has 1 to ${MAX_MUTEX_KEY_LENGTH} printable ASCII characters, without spaces at `
+                + `the ends: ${JSON.stringify(key)}`,
+            );
+        }
+    }
+
+    const unique = [...new Set(/** @type {string[]} */ (keys))].toSorted();
+
+    if (unique.length > MAX_MUTEX_KEYS) {
+        throw new InvalidMutexKeyError(`${where}: at most ${MAX_MUTEX_KEYS} mutex keys, got ${unique.length}`);
+    }
+
+    return unique;
+}

package/src/new_job.js

@@ -0,0 +1,128 @@
+import { monotonicFactory } from 'ulid';
+import { parseDuration } from './duration.js';
+import { InvalidOptionsError } from './errors.js';
+import { normalizeMutexKeys } from './mutex_keys.js';
+import { serializePayload } from './payload.js';
+import {
+    checkFailureJob,
+    checkJobFields,
+    checkObject,
+    JOB_NAME_REGEXP,
+    PRINTABLE_ID_REGEXP,
+} from './validate.js';
+
+const ENQUEUE_OPTION_NAMES = [
+    'jobId', 'delay', 'runAt', 'maxAttempts', 'maxStalls', 'retention', 'failureJob', 'mutexKeys',
+];
+const MAX_JOB_ID_LENGTH = 120;
+/** Sorted by creation order in this process, also within one millisecond */
+const ulid = monotonicFactory();
+
+/**
+ * A job to insert, with every value frozen at enqueue.
+ * @typedef {object} NewJob
+ * @property {string} id `<name>:<jobId>`, or a ULID.
+ * @property {boolean} hasJobId
+ * @property {string} name
+ * @property {string} payloadJson
+ * @property {readonly string[]} mutexKeys Sorted, without duplicates.
+ * @property {number} maxAttempts
+ * @property {number} maxStalls
+ * @property {number} completedRetentionMs
+ * @property {number} failedRetentionMs
+ * @property {string | null} failureJob
+ * @property {number} delayMs From the DB clock at insert. 0 with `runAt`.
+ * @property {Date | null} runAt
+ */
+
+/**
+ * @typedef {object} NewJobParams
+ * @property {string} name
+ * @property {import('./types.js').JobDefinition | null} definition Null: an unknown name, the defaults only.
+ * @property {import('./defaults.js').ResolvedDefaults} defaults
+ * @property {unknown} payload
+ * @property {import('./types.js').EnqueueOptions | undefined} options
+ * @property {number} maxPayloadBytes
+ */
+
+/**
+ * @param {unknown} jobId
+ * @param {string} where
+ */
+function checkJobId(jobId, where) {
+    if (typeof jobId !== 'string' || jobId.length > MAX_JOB_ID_LENGTH || !PRINTABLE_ID_REGEXP.test(jobId)) {
+        throw new InvalidOptionsError(
+            `${where}: "jobId" must have 1 to ${MAX_JOB_ID_LENGTH} printable ASCII characters, without spaces at the `
+            + `ends: ${JSON.stringify(jobId)}`,
+        );
+    }
+}
+
+/**
+ * @param {Record<string, unknown>} options
+ * @param {string} where
+ * @returns {{ delayMs: number, runAt: Date | null }}
+ */
+function getSchedule(options, where) {
+    if (options.delay !== undefined && options.runAt !== undefined) {
+        throw new InvalidOptionsError(`${where}: "delay" or "runAt", not both`);
+    }
+
+    if (options.runAt !== undefined) {
+        if (!(options.runAt instanceof Date) || Number.isNaN(options.runAt.getTime())) {
+            throw new InvalidOptionsError(`${where}: "runAt" must be a valid Date`);
+        }
+
+        return { delayMs: 0, runAt: options.runAt };
+    }
+
+    const delayMs = options.delay === undefined ? 0 : parseDuration(options.delay);
+
+    if (delayMs === null) {
+        throw new InvalidOptionsError(`${where}: "delay" must be a duration, got ${JSON.stringify(options.delay)}`);
+    }
+
+    return { delayMs, runAt: null };
+}
+
+/**
+ * Checks an enqueue and computes the job to insert: options, then the definition, then the defaults.
+ * @param {NewJobParams} params
+ * @returns {NewJob}
+ */
+export function buildNewJob({ name, definition, defaults, payload, options = {}, maxPayloadBytes }) {
+    const where = `enqueue(${JSON.stringify(name)})`;
+
+    if (typeof name !== 'string' || !JOB_NAME_REGEXP.test(name)) {
+        throw new InvalidOptionsError(`${where}: the job name must match ${JOB_NAME_REGEXP}`);
+    }
+
+    const scope = { where, ErrorClass: InvalidOptionsError };
+
+    checkObject(options, ENQUEUE_OPTION_NAMES, 'options', scope);
+    checkJobFields(options, scope);
+
+    if (options.jobId !== undefined) {
+        checkJobId(options.jobId, where);
+    }
+
+    const failureJob = options.failureJob ?? definition?.failureJob;
+    const payloadJson = serializePayload(payload, maxPayloadBytes, where);
+    // The keys come from the payload as the handler gets it: as JSON gives it back.
+    const rawKeys = options.mutexKeys ?? definition?.mutexKeys?.(JSON.parse(payloadJson)) ?? [];
+    const retention = { ...definition?.retention, ...options.retention };
+
+    return {
+        id: options.jobId === undefined ? ulid() : `${name}:${options.jobId}`,
+        hasJobId: options.jobId !== undefined,
+        name,
+        payloadJson,
+        mutexKeys: normalizeMutexKeys(rawKeys, where),
+        maxAttempts: options.maxAttempts ?? definition?.maxAttempts ?? defaults.maxAttempts,
+        maxStalls: options.maxStalls ?? definition?.maxStalls ?? defaults.maxStalls,
+        completedRetentionMs: parseDuration(retention.completed) ?? defaults.retention.completedMs,
+        failedRetentionMs: parseDuration(retention.failed) ?? defaults.retention.failedMs,
+        failureJob: failureJob === undefined ? null : checkFailureJob(failureJob, name, scope),
+        ...getSchedule(options, where),
+    };
+}

package/src/payload.js

@@ -0,0 +1,54 @@
+import { InvalidOptionsError, PayloadTooLargeError } from './errors.js';
+
+export const DEFAULT_MAX_PAYLOAD_BYTES = 256 * 1024;
+
+/**
+ * Serializes a payload: JSON, well-formed strings (MySQL refuses a lone surrogate), no NUL character (PostgreSQL
+ * refuses it), at most maxBytes.
+ * @param {unknown} payload
+ * @param {number} maxBytes
+ * @param {string} where For the error message.
+ * @returns {string} The JSON.
+ */
+export function serializePayload(payload, maxBytes, where) {
+    let wellFormed = true;
+    let hasNul = false;
+    let json;
+
+    try {
+        json = JSON.stringify(payload, (key, value) => {
+            if (!key.isWellFormed() || (typeof value === 'string' && !value.isWellFormed())) {
+                wellFormed = false;
+            }
+
+            if (key.includes('\0') || (typeof value === 'string' && value.includes('\0'))) {
+                hasNul = true;
+            }
+
+            return value;
+        });
+    } catch (error) {
+        throw new InvalidOptionsError(`${where}: the payload is not JSON`, { cause: error });
+    }
+
+    if (json === undefined) {
+        throw new InvalidOptionsError(`${where}: the payload is not JSON`);
+    }
+
+    if (!wellFormed) {
+        throw new InvalidOptionsError(`${where}: the payload has a string that is not well-formed UTF-16`);
+    }
+
+    if (hasNul) {
+        // PostgreSQL jsonb refuses it: refused on both databases, for the same behavior
+        throw new InvalidOptionsError(`${where}: the payload has a NUL character (\\u0000)`);
+    }
+
+    const bytes = Buffer.byteLength(json);
+
+    if (bytes > maxBytes) {
+        throw new PayloadTooLargeError(`${where}: the payload has ${bytes} bytes of JSON, at most ${maxBytes}`);
+    }
+
+    return json;
+}

package/src/registry.js

@@ -0,0 +1,169 @@
+// The handlers of an instance, their lanes and the known definitions.
+import { parseDuration } from './duration.js';
+import { ConfigError } from './errors.js';
+import { checkDuration, checkInteger, checkObject, isPlainObject, JOB_NAME_REGEXP } from './validate.js';
+
+/** @typedef {import('./types.js').JobDefinition} JobDefinition */
+
+/**
+ * A concurrency limit in this process.
+ * @typedef {object} Lane
+ * @property {string} name
+ * @property {number} concurrency
+ * @property {number} running
+ */
+
+/**
+ * @typedef {object} HandlerEntry
+ * @property {string} name
+ * @property {JobDefinition | null} definition
+ * @property {import('./types.js').Handler} handler
+ * @property {Lane} lane
+ * @property {number | null} timeoutMs Resolved: the handle() options, else the definition, else the defaults.
+ */
+
+/**
+ * @typedef {object} Registry
+ * @property {(job: JobDefinition | string, handler: import('./types.js').Handler, options?: object) => void} handle
+ * @property {(name: string) => HandlerEntry | undefined} getEntry
+ * @property {() => HandlerEntry[]} entries
+ * @property {(name: string) => JobDefinition | null} findDefinition In `jobs`, else handled.
+ * @property {() => string[]} keyedNames The handled names whose definition has mutex keys.
+ * @property {() => void} validate Checks the handlers together: each failure job resolves and has none of its own.
+ */
+
+/**
+ * @param {object} params
+ * @param {Map<string, JobDefinition>} params.jobsByName
+ * @param {unknown} params.lanes The `lanes` option.
+ * @param {number} params.concurrency The process concurrency.
+ * @param {number | null} params.defaultTimeoutMs
+ * @returns {Registry}
+ */
+export function createRegistry({ jobsByName, lanes, concurrency, defaultTimeoutMs }) {
+    const scope = { where: 'createSqlane', ErrorClass: ConfigError };
+    /** @type {Map<string, Lane>} */
+    const namedLanes = new Map();
+    /** The handlers without a lane: the process concurrency only */
+    const defaultLane = { name: '', concurrency, running: 0 };
+    /** @type {Map<string, HandlerEntry>} */
+    const entriesByName = new Map();
+
+    if (lanes !== undefined && !isPlainObject(lanes)) {
+        throw new ConfigError('createSqlane: "lanes" must be an object of lane options');
+    }
+
+    for (const [name, options] of Object.entries(lanes ?? {})) {
+        const laneOptions = checkObject(options, ['concurrency'], `lane "${name}" options`, scope);
+
+        checkInteger(laneOptions.concurrency, `lane "${name}" concurrency`, scope);
+        namedLanes.set(name, { name, concurrency: Math.min(Number(laneOptions.concurrency), concurrency), running: 0 });
+    }
+
+    /** @type {Registry['findDefinition']} */
+    function findDefinition(name) {
+        return jobsByName.get(name) ?? entriesByName.get(name)?.definition ?? null;
+    }
+
+    /** @type {Registry['handle']} */
+    function handle(job, handler, options = {}) {
+        const name = isPlainObject(job) ? job.name : job;
+        const where = `handle(${JSON.stringify(name)})`;
+        const handleScope = { where, ErrorClass: ConfigError };
+
+        if (typeof name !== 'string' || !JOB_NAME_REGEXP.test(name)) {
+            throw new ConfigError(`${where}: the job must be a job definition or a job name`);
+        }
+
+        if (typeof handler !== 'function') {
+            throw new ConfigError(`${where}: the handler must be a function`);
+        }
+
+        const handleOptions = checkObject(options, ['concurrency', 'lane', 'timeout'], 'options', handleScope);
+
+        if (entriesByName.has(name)) {
+            throw new ConfigError(`${where}: the job is handled already`);
+        }
+
+        const known = jobsByName.get(name);
+        const definition = isPlainObject(job) ? /** @type {JobDefinition} */ (job) : known ?? null;
+
+        if (known && definition !== known) {
+            throw new ConfigError(`${where}: another definition of "${name}" is in "jobs"`);
+        }
+
+        if (handleOptions.lane !== undefined && handleOptions.concurrency !== undefined) {
+            throw new ConfigError(`${where}: "lane" or "concurrency", not both`);
+        }
+
+        /** @type {Lane} */
+        let lane = defaultLane;
+
+        if (handleOptions.lane !== undefined) {
+            const named = namedLanes.get(String(handleOptions.lane));
+
+            if (!named) {
+                throw new ConfigError(`${where}: unknown lane ${JSON.stringify(handleOptions.lane)}`);
+            }
+
+            lane = named;
+        } else if (handleOptions.concurrency !== undefined) {
+            checkInteger(handleOptions.concurrency, '"concurrency"', handleScope);
+            lane = {
+                name: `handler:${name}`,
+                concurrency: Math.min(Number(handleOptions.concurrency), concurrency),
+                running: 0,
+            };
+        }
+
+        if (handleOptions.timeout !== undefined) {
+            checkDuration(handleOptions.timeout, '"timeout"', handleScope, true);
+        }
+
+        const timeoutMs = parseDuration(handleOptions.timeout ?? definition?.timeout) ?? defaultTimeoutMs;
+
+        if (definition?.mutexKeys && timeoutMs === null) {
+            throw new ConfigError(
+                `${where}: a job with mutex keys needs a timeout (in handle(), the definition or "defaults")`,
+            );
+        }
+
+        entriesByName.set(name, { name, definition, handler, lane, timeoutMs });
+    }
+
+    /** @type {Registry['validate']} */
+    function validate() {
+        for (const entry of entriesByName.values()) {
+            const failureJob = entry.definition?.failureJob;
+
+            if (failureJob === undefined) {
+                continue;
+            }
+
+            const where = `handle(${JSON.stringify(entry.name)})`;
+            const known = typeof failureJob !== 'string' || jobsByName.has(failureJob) || entriesByName.has(failureJob);
+
+            if (!known) {
+                throw new ConfigError(`${where}: the failure job "${failureJob}" is not in "jobs" and not handled`);
+            }
+
+            // Handled by name only: no definition, so no failure job of its own
+            const failureDefinition = typeof failureJob === 'string' ? findDefinition(failureJob) : failureJob;
+
+            if (failureDefinition?.failureJob !== undefined) {
+                throw new ConfigError(`${where}: the failure job "${failureDefinition?.name}" has a failure job of its own`);
+            }
+        }
+    }
+
+    return {
+        handle,
+        getEntry: name => entriesByName.get(name),
+        entries: () => [...entriesByName.values()],
+        findDefinition,
+        keyedNames: () => [...entriesByName.values()]
+            .filter(entry => entry.definition?.mutexKeys)
+            .map(entry => entry.name),
+        validate,
+    };
+}