@sqlane/core 0.0.2 → 0.1.0

package/src/enqueue.js

@@ -0,0 +1,132 @@
+import { monotonicFactory } from 'ulid';
+import { InvalidOptionsError, UnknownJobError } from './errors.js';
+import { buildNewJob } from './new_job.js';
+import { insertJobsMysql } from './store/mysql_enqueue.js';
+import { insertJobsPostgres } from './store/postgres_enqueue.js';
+import { isPlainObject } from './validate.js';
+
+const enqueueToken = monotonicFactory();
+
+/**
+ * @template C
+ * @typedef {object} EnqueuerOptions
+ * @property {import('./types.js').Adapter<C>} adapter
+ * @property {string} tablePrefix Checked.
+ * @property {(name: string) => import('./types.js').JobDefinition | null} findDefinition The known definition of
+ *     a name.
+ * @property {'allow' | 'reject'} unknownJobs
+ * @property {import('./defaults.js').ResolvedDefaults} defaults
+ * @property {number} maxPayloadBytes
+ * @property {string | null} notifyChannel PostgreSQL NOTIFY: `pg_notify` in the enqueue transaction, delivered on
+ *     commit. Null: no NOTIFY.
+ */
+
+/**
+ * @template C
+ * @typedef {object} Enqueuer
+ * @property {import('./types.js').Sqlane<C, never>['enqueue']} enqueue
+ * @property {import('./types.js').Sqlane<C, never>['enqueueMany']} enqueueMany
+ */
+
+/**
+ * The enqueue side of an instance.
+ * @template C
+ * @param {EnqueuerOptions<C>} options
+ * @returns {Enqueuer<C>}
+ */
+export function createEnqueuer({
+    adapter,
+    tablePrefix,
+    findDefinition,
+    unknownJobs,
+    defaults,
+    maxPayloadBytes,
+    notifyChannel,
+}) {
+    const insertJobs = adapter.dialect === 'mysql' ? insertJobsMysql : insertJobsPostgres;
+
+    /**
+     * @param {import('./types.js').JobDefinition | string} job
+     * @returns {{ name: string, definition: import('./types.js').JobDefinition | null }}
+     */
+    function resolveJob(job) {
+        if (isPlainObject(job) && typeof job.name === 'string') {
+            return { name: job.name, definition: job };
+        }
+
+        if (typeof job !== 'string') {
+            throw new InvalidOptionsError('enqueue: the job must be a job definition or a job name');
+        }
+
+        const definition = findDefinition(job);
+
+        if (!definition && unknownJobs === 'reject') {
+            throw new UnknownJobError(`enqueue("${job}"): the job is not in "jobs" and not handled`);
+        }
+
+        return { name: job, definition };
+    }
+
+    /**
+     * @param {C} connection
+     * @param {import('./types.js').JobDefinition | string} job
+     * @param {readonly { payload: unknown, options?: import('./types.js').EnqueueOptions }[]} items
+     * @returns {Promise<import('./types.js').EnqueueResult[]>}
+     */
+    async function insert(connection, job, items) {
+        const { name, definition } = resolveJob(job);
+        // Checks all the items before the INSERT: a bad item adds no job.
+        const newJobs = items.map(item => buildNewJob({
+            name,
+            definition,
+            defaults,
+            payload: item.payload,
+            options: item.options,
+            maxPayloadBytes,
+        }));
+        // A jobId repeated in the call: its first item is the job, the next ones are duplicates.
+        const firstById = new Map();
+
+        for (const newJob of newJobs) {
+            if (!firstById.has(newJob.id)) {
+                firstById.set(newJob.id, newJob);
+            }
+        }
+
+        if (!firstById.size) {
+            return [];
+        }
+
+        const token = enqueueToken();
+        const created = await adapter.inTransaction(connection, async (executor) => {
+            const createdIds = await insertJobs(executor, tablePrefix, [...firstById.values()], token);
+
+            if (notifyChannel && createdIds.size) {
+                // One NOTIFY per transaction and name: PostgreSQL drops the repeated ones
+                await executor.query('SELECT pg_notify($1, $2)', [notifyChannel, name]);
+            }
+
+            return createdIds;
+        });
+
+        return newJobs.map(newJob => ({
+            id: newJob.id,
+            created: created.has(newJob.id) && firstById.get(newJob.id) === newJob,
+        }));
+    }
+
+    return {
+        enqueue: async (connection, job, payload, options) => {
+            const [result] = await insert(connection, job, [{ payload, options }]);
+
+            return result;
+        },
+        enqueueMany: (connection, job, items) => {
+            if (!Array.isArray(items)) {
+                return Promise.reject(new InvalidOptionsError('enqueueMany: "items" must be an array'));
+            }
+
+            return insert(connection, job, items);
+        },
+    };
+}

package/src/error_json.js

@@ -0,0 +1,30 @@
+/** The longest stack kept with a job error: the error row and the failure job stay small. */
+export const MAX_STACK_LENGTH = 8 * 1024;
+
+/**
+ * @typedef {object} ErrorJson
+ * @property {string} name
+ * @property {string} message
+ * @property {string} [stack]
+ * @property {string} [code]
+ */
+
+/**
+ * The JSON of an error, for `last_error` and the failure job. A thrown non-Error becomes its string.
+ * @param {unknown} error
+ * @returns {ErrorJson}
+ */
+export function toErrorJson(error) {
+    if (!(error instanceof Error)) {
+        return { name: 'Error', message: String(error) };
+    }
+
+    const { code } = /** @type {{ code?: unknown }} */ (error);
+
+    return {
+        name: error.name,
+        message: error.message,
+        ...(error.stack && { stack: error.stack.slice(0, MAX_STACK_LENGTH) }),
+        ...(typeof code === 'string' && { code }),
+    };
+}

package/src/errors.js

@@ -174,3 +174,15 @@ export class RetryJobError extends SqlaneError {
         this.delay = delay;
     }
 }
+
+/** `migrate()` did not get the migration lock in its `lockTimeout`. */
+export class MigrationLockTimeoutError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('MIGRATION_LOCK_TIMEOUT', message, options);
+        this.name = 'MigrationLockTimeoutError';
+    }
+}

package/src/hint_channel.js

@@ -0,0 +1,107 @@
+// Hints between processes: "jobs of this name are ready". Only an optimization: a lost hint delays a job until the
+// next poll, it never loses one.
+
+/** At most one message per name in this window; the messages in it make one more at its end */
+const THROTTLE_MS = 100;
+/** The message after a release of mutex keys: every process probes all its names */
+export const ANY_NAME = '*';
+
+/**
+ * @typedef {object} HintChannel
+ * @property {(name: string) => void} publish Fire-and-forget: never throws, never waits.
+ * @property {(onMessage: (name: string) => void) => Promise<void>} listen
+ * @property {() => Promise<void>} stop Stops listening; drops the pending messages.
+ */
+
+/**
+ * @param {object} params
+ * @param {import('./types.js').Hint | null} params.hint
+ * @param {string} params.channel
+ * @param {(operation: string, error: unknown) => void} params.reportError
+ * @returns {HintChannel}
+ */
+export function createHintChannel({ hint, channel, reportError }) {
+    /** @type {Map<string, { timer: NodeJS.Timeout, again: boolean }>} */
+    const windows = new Map();
+    /** @type {(() => Promise<void>) | null} */
+    let unsubscribe = null;
+    let stopped = false;
+
+    /** @param {string} name */
+    function send(name) {
+        if (!hint || stopped) {
+            return;
+        }
+
+        const { publish } = hint;
+
+        // Not awaited: the enqueue does not wait for the hint
+        (async () => {
+            try {
+                await publish(channel, name);
+            } catch (error) {
+                reportError('hint_publish', error);
+            }
+        })();
+    }
+
+    /** @param {string} name */
+    function openWindow(name) {
+        const timer = setTimeout(() => {
+            const window = windows.get(name);
+
+            windows.delete(name);
+
+            if (window?.again) {
+                send(name);
+                openWindow(name);
+            }
+        }, THROTTLE_MS);
+
+        timer.unref();
+        windows.set(name, { timer, again: false });
+    }
+
+    return {
+        publish: (name) => {
+            const window = windows.get(name);
+
+            if (window) {
+                window.again = true;
+
+                return;
+            }
+
+            send(name);
+            openWindow(name);
+        },
+        listen: async (onMessage) => {
+            if (!hint) {
+                return;
+            }
+
+            unsubscribe = await hint.subscribe(channel, (message) => {
+                try {
+                    onMessage(message);
+                } catch (error) {
+                    reportError('hint_message', error);
+                }
+            });
+        },
+        stop: async () => {
+            stopped = true;
+
+            for (const { timer } of windows.values()) {
+                clearTimeout(timer);
+            }
+
+            windows.clear();
+
+            try {
+                await unsubscribe?.();
+            } catch (error) {
+                reportError('hint_unsubscribe', error);
+            }
+        },
+    };
+}

package/src/index.js

@@ -11,6 +11,7 @@ export {
     InvalidOptionsError,
     JobTimeoutError,
     LeaseLostError,
+    MigrationLockTimeoutError,
     MissingPayloadError,
     PayloadTooLargeError,
     RetryJobError,

package/src/migrations.js

@@ -1,3 +1,11 @@
+import { parseDuration } from './duration.js';
+import { ConfigError, InvalidOptionsError, MigrationLockTimeoutError } from './errors.js';
+import { mysqlMigrations, mysqlMigrationsTable } from './schema/mysql.js';
+import { postgresMigrations, postgresMigrationsTable } from './schema/postgres.js';
+import { checkTablePrefix } from './validate.js';
+
+/** @typedef {import('./types.js').Executor} Executor */
+
 /**
  * The schema of sqlane as versioned SQL, for the app's own migration tool.
  * @param {object} options
@@ -5,6 +13,151 @@
  * @param {string} [options.tablePrefix] Default: `'sqlane_'`.
  * @returns {import('./types.js').Migration[]}
  */
-export function getMigrations(options) {
-    throw new Error(`sqlane: getMigrations(${options.dialect}) is not implemented yet`);
+export function getMigrations({ dialect, tablePrefix = 'sqlane_' }) {
+    const scope = { where: 'getMigrations', ErrorClass: ConfigError };
+
+    checkTablePrefix(tablePrefix, scope);
+
+    if (dialect === 'mysql') {
+        return mysqlMigrations(tablePrefix);
+    }
+
+    if (dialect === 'postgres') {
+        return postgresMigrations(tablePrefix);
+    }
+
+    throw new ConfigError(`getMigrations: "dialect" must be 'mysql' or 'postgres'`);
+}
+
+/**
+ * Runs the statements one after the other: DDL in order.
+ * @param {Executor} executor
+ * @param {{ sql: string, params: unknown[] }[]} steps
+ * @returns {Promise<void>}
+ */
+async function runInOrder(executor, steps) {
+    for (const { sql, params } of steps) {
+        // oxlint-disable-next-line no-await-in-loop -- each statement needs the previous one
+        await executor.query(sql, params);
+    }
+}
+
+/**
+ * The statements of the missing migrations, each followed by its record.
+ * @param {import('./types.js').Migration[]} missing
+ * @param {string} recordSql
+ * @returns {{ sql: string, params: unknown[] }[]}
+ */
+function toSteps(missing, recordSql) {
+    return missing.flatMap(({ id, statements }) => [
+        ...statements.map(sql => ({ sql, params: [] })),
+        { sql: recordSql, params: [id] },
+    ]);
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} table
+ * @returns {Promise<Set<string>>}
+ */
+async function readApplied(executor, table) {
+    const rows = await executor.query(`SELECT id FROM ${table}`, []);
+
+    return new Set(rows.map(row => String(row.id)));
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} lockTimeoutMs
+ * @returns {Promise<string[]>}
+ */
+async function migrateMysql(executor, p, lockTimeoutMs) {
+    const lockName = `${p}migrations`;
+    // GET_LOCK waits whole seconds; -1 is forever, so at least 1
+    const [{ locked }] = await executor.query('SELECT GET_LOCK(?, ?) AS locked', [
+        lockName,
+        Math.max(1, Math.ceil(lockTimeoutMs / 1000)),
+    ]);
+
+    if (Number(locked) !== 1) {
+        throw new MigrationLockTimeoutError(`migrate: the lock "${lockName}" was not free in ${lockTimeoutMs} ms`);
+    }
+
+    try {
+        await executor.query(mysqlMigrationsTable(p), []);
+
+        const applied = await readApplied(executor, `${p}migrations`);
+        const missing = mysqlMigrations(p).filter(({ id }) => !applied.has(id));
+
+        await runInOrder(executor, toSteps(missing, `INSERT INTO ${p}migrations (id, applied_at) VALUES (?, UTC_TIMESTAMP(3))`));
+
+        return missing.map(({ id }) => id);
+    } finally {
+        await executor.query('SELECT RELEASE_LOCK(?)', [lockName]);
+    }
+}
+
+/**
+ * @param {Executor} executor
+ * @param {string} p
+ * @param {number} lockTimeoutMs
+ * @returns {Promise<string[]>}
+ */
+async function migratePostgres(executor, p, lockTimeoutMs) {
+    await executor.query('BEGIN', []);
+
+    try {
+        // SET LOCAL: until the end of the transaction, then the pooled connection gets its settings back
+        await executor.query('SET LOCAL statement_timeout = 0', []);
+        await executor.query(`SET LOCAL lock_timeout = ${Math.max(1, lockTimeoutMs)}`, []);
+
+        try {
+            await executor.query('SELECT pg_advisory_xact_lock(hashtext($1))', [`${p}migrations`]);
+        } catch (error) {
+            if (/** @type {{ code?: string }} */ (error).code === '55P03') {
+                throw new MigrationLockTimeoutError(
+                    `migrate: the lock of "${p}migrations" was not free in ${lockTimeoutMs} ms`,
+                    { cause: error },
+                );
+            }
+
+            throw error;
+        }
+
+        await executor.query('SET LOCAL lock_timeout = 0', []);
+        await executor.query(postgresMigrationsTable(p), []);
+
+        const applied = await readApplied(executor, `${p}migrations`);
+        const missing = postgresMigrations(p).filter(({ id }) => !applied.has(id));
+
+        await runInOrder(executor, toSteps(missing, `INSERT INTO ${p}migrations (id, applied_at) VALUES ($1, clock_timestamp())`));
+
+        await executor.query('COMMIT', []);
+
+        return missing.map(({ id }) => id);
+    } catch (error) {
+        await executor.query('ROLLBACK', []);
+        throw error;
+    }
+}
+
+/**
+ * Applies the missing migrations under a DB lock, on a connection of its own, without a statement timeout.
+ * @template C
+ * @param {import('./types.js').Adapter<C>} adapter
+ * @param {string} tablePrefix Checked.
+ * @param {import('./types.js').MigrateOptions} [options]
+ * @returns {Promise<string[]>} The ids of the applied migrations.
+ */
+export function runMigrations(adapter, tablePrefix, options = {}) {
+    const lockTimeoutMs = parseDuration(options.lockTimeout ?? '10m');
+
+    if (lockTimeoutMs === null) {
+        return Promise.reject(new InvalidOptionsError('migrate: "lockTimeout" must be a duration'));
+    }
+
+    return adapter.withConnection(executor => (adapter.dialect === 'mysql'
+        ? migrateMysql(executor, tablePrefix, lockTimeoutMs)
+        : migratePostgres(executor, tablePrefix, lockTimeoutMs)));
 }

package/src/mutex_keys.js

@@ -0,0 +1,34 @@
+import { InvalidMutexKeyError } from './errors.js';
+import { PRINTABLE_ID_REGEXP } from './validate.js';
+
+export const MAX_MUTEX_KEYS = 10;
+export const MAX_MUTEX_KEY_LENGTH = 255;
+
+/**
+ * Checks the mutex keys of a job.
+ * @param {unknown} keys
+ * @param {string} where For the error message.
+ * @returns {string[]} The keys without duplicates, sorted (the lock order).
+ */
+export function normalizeMutexKeys(keys, where) {
+    if (!Array.isArray(keys)) {
+        throw new InvalidMutexKeyError(`${where}: the mutex keys must be an array`);
+    }
+
+    for (const key of keys) {
+        if (typeof key !== 'string' || key.length > MAX_MUTEX_KEY_LENGTH || !PRINTABLE_ID_REGEXP.test(key)) {
+            throw new InvalidMutexKeyError(
+                `${where}: a mutex key has 1 to ${MAX_MUTEX_KEY_LENGTH} printable ASCII characters, without spaces at `
+                + `the ends: ${JSON.stringify(key)}`,
+            );
+        }
+    }
+
+    const unique = [...new Set(/** @type {string[]} */ (keys))].toSorted();
+
+    if (unique.length > MAX_MUTEX_KEYS) {
+        throw new InvalidMutexKeyError(`${where}: at most ${MAX_MUTEX_KEYS} mutex keys, got ${unique.length}`);
+    }
+
+    return unique;
+}

package/src/new_job.js

@@ -0,0 +1,128 @@
+import { monotonicFactory } from 'ulid';
+import { parseDuration } from './duration.js';
+import { InvalidOptionsError } from './errors.js';
+import { normalizeMutexKeys } from './mutex_keys.js';
+import { serializePayload } from './payload.js';
+import {
+    checkFailureJob,
+    checkJobFields,
+    checkObject,
+    JOB_NAME_REGEXP,
+    PRINTABLE_ID_REGEXP,
+} from './validate.js';
+
+const ENQUEUE_OPTION_NAMES = [
+    'jobId', 'delay', 'runAt', 'maxAttempts', 'maxStalls', 'retention', 'failureJob', 'mutexKeys',
+];
+const MAX_JOB_ID_LENGTH = 120;
+/** Sorted by creation order in this process, also within one millisecond */
+const ulid = monotonicFactory();
+
+/**
+ * A job to insert, with every value frozen at enqueue.
+ * @typedef {object} NewJob
+ * @property {string} id `<name>:<jobId>`, or a ULID.
+ * @property {boolean} hasJobId
+ * @property {string} name
+ * @property {string} payloadJson
+ * @property {readonly string[]} mutexKeys Sorted, without duplicates.
+ * @property {number} maxAttempts
+ * @property {number} maxStalls
+ * @property {number} completedRetentionMs
+ * @property {number} failedRetentionMs
+ * @property {string | null} failureJob
+ * @property {number} delayMs From the DB clock at insert. 0 with `runAt`.
+ * @property {Date | null} runAt
+ */
+
+/**
+ * @typedef {object} NewJobParams
+ * @property {string} name
+ * @property {import('./types.js').JobDefinition | null} definition Null: an unknown name, the defaults only.
+ * @property {import('./defaults.js').ResolvedDefaults} defaults
+ * @property {unknown} payload
+ * @property {import('./types.js').EnqueueOptions | undefined} options
+ * @property {number} maxPayloadBytes
+ */
+
+/**
+ * @param {unknown} jobId
+ * @param {string} where
+ */
+function checkJobId(jobId, where) {
+    if (typeof jobId !== 'string' || jobId.length > MAX_JOB_ID_LENGTH || !PRINTABLE_ID_REGEXP.test(jobId)) {
+        throw new InvalidOptionsError(
+            `${where}: "jobId" must have 1 to ${MAX_JOB_ID_LENGTH} printable ASCII characters, without spaces at the `
+            + `ends: ${JSON.stringify(jobId)}`,
+        );
+    }
+}
+
+/**
+ * @param {Record<string, unknown>} options
+ * @param {string} where
+ * @returns {{ delayMs: number, runAt: Date | null }}
+ */
+function getSchedule(options, where) {
+    if (options.delay !== undefined && options.runAt !== undefined) {
+        throw new InvalidOptionsError(`${where}: "delay" or "runAt", not both`);
+    }
+
+    if (options.runAt !== undefined) {
+        if (!(options.runAt instanceof Date) || Number.isNaN(options.runAt.getTime())) {
+            throw new InvalidOptionsError(`${where}: "runAt" must be a valid Date`);
+        }
+
+        return { delayMs: 0, runAt: options.runAt };
+    }
+
+    const delayMs = options.delay === undefined ? 0 : parseDuration(options.delay);
+
+    if (delayMs === null) {
+        throw new InvalidOptionsError(`${where}: "delay" must be a duration, got ${JSON.stringify(options.delay)}`);
+    }
+
+    return { delayMs, runAt: null };
+}
+
+/**
+ * Checks an enqueue and computes the job to insert: options, then the definition, then the defaults.
+ * @param {NewJobParams} params
+ * @returns {NewJob}
+ */
+export function buildNewJob({ name, definition, defaults, payload, options = {}, maxPayloadBytes }) {
+    const where = `enqueue(${JSON.stringify(name)})`;
+
+    if (typeof name !== 'string' || !JOB_NAME_REGEXP.test(name)) {
+        throw new InvalidOptionsError(`${where}: the job name must match ${JOB_NAME_REGEXP}`);
+    }
+
+    const scope = { where, ErrorClass: InvalidOptionsError };
+
+    checkObject(options, ENQUEUE_OPTION_NAMES, 'options', scope);
+    checkJobFields(options, scope);
+
+    if (options.jobId !== undefined) {
+        checkJobId(options.jobId, where);
+    }
+
+    const failureJob = options.failureJob ?? definition?.failureJob;
+    const payloadJson = serializePayload(payload, maxPayloadBytes, where);
+    // The keys come from the payload as the handler gets it: as JSON gives it back.
+    const rawKeys = options.mutexKeys ?? definition?.mutexKeys?.(JSON.parse(payloadJson)) ?? [];
+    const retention = { ...definition?.retention, ...options.retention };
+
+    return {
+        id: options.jobId === undefined ? ulid() : `${name}:${options.jobId}`,
+        hasJobId: options.jobId !== undefined,
+        name,
+        payloadJson,
+        mutexKeys: normalizeMutexKeys(rawKeys, where),
+        maxAttempts: options.maxAttempts ?? definition?.maxAttempts ?? defaults.maxAttempts,
+        maxStalls: options.maxStalls ?? definition?.maxStalls ?? defaults.maxStalls,
+        completedRetentionMs: parseDuration(retention.completed) ?? defaults.retention.completedMs,
+        failedRetentionMs: parseDuration(retention.failed) ?? defaults.retention.failedMs,
+        failureJob: failureJob === undefined ? null : checkFailureJob(failureJob, name, scope),
+        ...getSchedule(options, where),
+    };
+}

package/src/payload.js

@@ -0,0 +1,54 @@
+import { InvalidOptionsError, PayloadTooLargeError } from './errors.js';
+
+export const DEFAULT_MAX_PAYLOAD_BYTES = 256 * 1024;
+
+/**
+ * Serializes a payload: JSON, well-formed strings (MySQL refuses a lone surrogate), no NUL character (PostgreSQL
+ * refuses it), at most maxBytes.
+ * @param {unknown} payload
+ * @param {number} maxBytes
+ * @param {string} where For the error message.
+ * @returns {string} The JSON.
+ */
+export function serializePayload(payload, maxBytes, where) {
+    let wellFormed = true;
+    let hasNul = false;
+    let json;
+
+    try {
+        json = JSON.stringify(payload, (key, value) => {
+            if (!key.isWellFormed() || (typeof value === 'string' && !value.isWellFormed())) {
+                wellFormed = false;
+            }
+
+            if (key.includes('\0') || (typeof value === 'string' && value.includes('\0'))) {
+                hasNul = true;
+            }
+
+            return value;
+        });
+    } catch (error) {
+        throw new InvalidOptionsError(`${where}: the payload is not JSON`, { cause: error });
+    }
+
+    if (json === undefined) {
+        throw new InvalidOptionsError(`${where}: the payload is not JSON`);
+    }
+
+    if (!wellFormed) {
+        throw new InvalidOptionsError(`${where}: the payload has a string that is not well-formed UTF-16`);
+    }
+
+    if (hasNul) {
+        // PostgreSQL jsonb refuses it: refused on both databases, for the same behavior
+        throw new InvalidOptionsError(`${where}: the payload has a NUL character (\\u0000)`);
+    }
+
+    const bytes = Buffer.byteLength(json);
+
+    if (bytes > maxBytes) {
+        throw new PayloadTooLargeError(`${where}: the payload has ${bytes} bytes of JSON, at most ${maxBytes}`);
+    }
+
+    return json;
+}