@sqlane/core 0.0.1 → 0.1.0

package/src/duration.js

@@ -0,0 +1,27 @@
+import parse from 'parse-duration';
+
+/**
+ * Parses a duration: milliseconds (a non-negative safe integer), or a string that parse-duration reads, such as
+ * `'500ms'`, `'5s'`, `'1.5h'`, `'4m30s'`, `'2 days'`.
+ * @param {unknown} value
+ * @returns {number | null} The milliseconds, rounded. Null: not a duration, or negative.
+ */
+export function parseDuration(value) {
+    if (typeof value === 'number') {
+        return Number.isSafeInteger(value) && value >= 0 ? value : null;
+    }
+
+    if (typeof value !== 'string') {
+        return null;
+    }
+
+    const ms = parse(value);
+
+    if (typeof ms !== 'number' || !(ms >= 0)) {
+        return null;
+    }
+
+    const rounded = Math.round(ms);
+
+    return Number.isSafeInteger(rounded) ? rounded : null;
+}

package/src/enqueue.js

@@ -0,0 +1,132 @@
+import { monotonicFactory } from 'ulid';
+import { InvalidOptionsError, UnknownJobError } from './errors.js';
+import { buildNewJob } from './new_job.js';
+import { insertJobsMysql } from './store/mysql_enqueue.js';
+import { insertJobsPostgres } from './store/postgres_enqueue.js';
+import { isPlainObject } from './validate.js';
+
+const enqueueToken = monotonicFactory();
+
+/**
+ * @template C
+ * @typedef {object} EnqueuerOptions
+ * @property {import('./types.js').Adapter<C>} adapter
+ * @property {string} tablePrefix Checked.
+ * @property {(name: string) => import('./types.js').JobDefinition | null} findDefinition The known definition of
+ *     a name.
+ * @property {'allow' | 'reject'} unknownJobs
+ * @property {import('./defaults.js').ResolvedDefaults} defaults
+ * @property {number} maxPayloadBytes
+ * @property {string | null} notifyChannel PostgreSQL NOTIFY: `pg_notify` in the enqueue transaction, delivered on
+ *     commit. Null: no NOTIFY.
+ */
+
+/**
+ * @template C
+ * @typedef {object} Enqueuer
+ * @property {import('./types.js').Sqlane<C, never>['enqueue']} enqueue
+ * @property {import('./types.js').Sqlane<C, never>['enqueueMany']} enqueueMany
+ */
+
+/**
+ * The enqueue side of an instance.
+ * @template C
+ * @param {EnqueuerOptions<C>} options
+ * @returns {Enqueuer<C>}
+ */
+export function createEnqueuer({
+    adapter,
+    tablePrefix,
+    findDefinition,
+    unknownJobs,
+    defaults,
+    maxPayloadBytes,
+    notifyChannel,
+}) {
+    const insertJobs = adapter.dialect === 'mysql' ? insertJobsMysql : insertJobsPostgres;
+
+    /**
+     * @param {import('./types.js').JobDefinition | string} job
+     * @returns {{ name: string, definition: import('./types.js').JobDefinition | null }}
+     */
+    function resolveJob(job) {
+        if (isPlainObject(job) && typeof job.name === 'string') {
+            return { name: job.name, definition: job };
+        }
+
+        if (typeof job !== 'string') {
+            throw new InvalidOptionsError('enqueue: the job must be a job definition or a job name');
+        }
+
+        const definition = findDefinition(job);
+
+        if (!definition && unknownJobs === 'reject') {
+            throw new UnknownJobError(`enqueue("${job}"): the job is not in "jobs" and not handled`);
+        }
+
+        return { name: job, definition };
+    }
+
+    /**
+     * @param {C} connection
+     * @param {import('./types.js').JobDefinition | string} job
+     * @param {readonly { payload: unknown, options?: import('./types.js').EnqueueOptions }[]} items
+     * @returns {Promise<import('./types.js').EnqueueResult[]>}
+     */
+    async function insert(connection, job, items) {
+        const { name, definition } = resolveJob(job);
+        // Checks all the items before the INSERT: a bad item adds no job.
+        const newJobs = items.map(item => buildNewJob({
+            name,
+            definition,
+            defaults,
+            payload: item.payload,
+            options: item.options,
+            maxPayloadBytes,
+        }));
+        // A jobId repeated in the call: its first item is the job, the next ones are duplicates.
+        const firstById = new Map();
+
+        for (const newJob of newJobs) {
+            if (!firstById.has(newJob.id)) {
+                firstById.set(newJob.id, newJob);
+            }
+        }
+
+        if (!firstById.size) {
+            return [];
+        }
+
+        const token = enqueueToken();
+        const created = await adapter.inTransaction(connection, async (executor) => {
+            const createdIds = await insertJobs(executor, tablePrefix, [...firstById.values()], token);
+
+            if (notifyChannel && createdIds.size) {
+                // One NOTIFY per transaction and name: PostgreSQL drops the repeated ones
+                await executor.query('SELECT pg_notify($1, $2)', [notifyChannel, name]);
+            }
+
+            return createdIds;
+        });
+
+        return newJobs.map(newJob => ({
+            id: newJob.id,
+            created: created.has(newJob.id) && firstById.get(newJob.id) === newJob,
+        }));
+    }
+
+    return {
+        enqueue: async (connection, job, payload, options) => {
+            const [result] = await insert(connection, job, [{ payload, options }]);
+
+            return result;
+        },
+        enqueueMany: (connection, job, items) => {
+            if (!Array.isArray(items)) {
+                return Promise.reject(new InvalidOptionsError('enqueueMany: "items" must be an array'));
+            }
+
+            return insert(connection, job, items);
+        },
+    };
+}

package/src/error_json.js

@@ -0,0 +1,30 @@
+/** The longest stack kept with a job error: the error row and the failure job stay small. */
+export const MAX_STACK_LENGTH = 8 * 1024;
+
+/**
+ * @typedef {object} ErrorJson
+ * @property {string} name
+ * @property {string} message
+ * @property {string} [stack]
+ * @property {string} [code]
+ */
+
+/**
+ * The JSON of an error, for `last_error` and the failure job. A thrown non-Error becomes its string.
+ * @param {unknown} error
+ * @returns {ErrorJson}
+ */
+export function toErrorJson(error) {
+    if (!(error instanceof Error)) {
+        return { name: 'Error', message: String(error) };
+    }
+
+    const { code } = /** @type {{ code?: unknown }} */ (error);
+
+    return {
+        name: error.name,
+        message: error.message,
+        ...(error.stack && { stack: error.stack.slice(0, MAX_STACK_LENGTH) }),
+        ...(typeof code === 'string' && { code }),
+    };
+}

package/src/errors.js

@@ -0,0 +1,188 @@
+/**
+ * The base of the errors of sqlane.
+ */
+export class SqlaneError extends Error {
+    /**
+     * @param {string} code A stable code, e.g. `CONFIG`.
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'SqlaneError';
+        /** @readonly */
+        this.code = code;
+    }
+}
+
+/** Bad `createSqlane`, `defineJob` or `handle()` options. */
+export class ConfigError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('CONFIG', message, options);
+        this.name = 'ConfigError';
+    }
+}
+
+/** Bad enqueue options: a bad name or `jobId`, `delay` with `runAt`. */
+export class InvalidOptionsError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('INVALID_OPTIONS', message, options);
+        this.name = 'InvalidOptionsError';
+    }
+}
+
+/** The payload is above `maxPayloadBytes`. */
+export class PayloadTooLargeError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('PAYLOAD_TOO_LARGE', message, options);
+        this.name = 'PayloadTooLargeError';
+    }
+}
+
+/** More than 10 mutex keys, or a key that breaks the rules. */
+export class InvalidMutexKeyError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('INVALID_MUTEX_KEY', message, options);
+        this.name = 'InvalidMutexKeyError';
+    }
+}
+
+/** Enqueue of an unknown name with `unknownJobs: 'reject'`. */
+export class UnknownJobError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('UNKNOWN_JOB', message, options);
+        this.name = 'UnknownJobError';
+    }
+}
+
+/** `start`, `handle` or `drain` after `close()`. */
+export class ClosedError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('CLOSED', message, options);
+        this.name = 'ClosedError';
+    }
+}
+
+/** `drain()` did not finish in its `timeout`. */
+export class DrainTimeoutError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('DRAIN_TIMEOUT', message, options);
+        this.name = 'DrainTimeoutError';
+    }
+}
+
+/** The `signal.reason` on a timeout; the error of the attempt. */
+export class JobTimeoutError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('JOB_TIMEOUT', message, options);
+        this.name = 'JobTimeoutError';
+    }
+}
+
+/** The `signal.reason` at the close deadline. */
+export class ShutdownError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('SHUTDOWN', message, options);
+        this.name = 'ShutdownError';
+    }
+}
+
+/** The `signal.reason` when the lease could not be renewed in time. */
+export class LeaseLostError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('LEASE_LOST', message, options);
+        this.name = 'LeaseLostError';
+    }
+}
+
+/** The error of a job without its payload. */
+export class MissingPayloadError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('MISSING_PAYLOAD', message, options);
+        this.name = 'MissingPayloadError';
+    }
+}
+
+/** Thrown by a handler: the job fails at once, without retries; its failure job runs. */
+export class FatalJobError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('FATAL_JOB', message, options);
+        this.name = 'FatalJobError';
+    }
+}
+
+/** Thrown by a handler: the job is retried after `delay` instead of the backoff. It still uses an attempt. */
+export class RetryJobError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions & { delay: import('./types.js').Duration }} options
+     */
+    constructor(message, options) {
+        const { delay, ...errorOptions } = options;
+
+        super('RETRY_JOB', message, errorOptions);
+        this.name = 'RetryJobError';
+        /** @readonly */
+        this.delay = delay;
+    }
+}
+
+/** `migrate()` did not get the migration lock in its `lockTimeout`. */
+export class MigrationLockTimeoutError extends SqlaneError {
+    /**
+     * @param {string} message
+     * @param {ErrorOptions} [options]
+     */
+    constructor(message, options) {
+        super('MIGRATION_LOCK_TIMEOUT', message, options);
+        this.name = 'MigrationLockTimeoutError';
+    }
+}

package/src/hint_channel.js

@@ -0,0 +1,107 @@
+// Hints between processes: "jobs of this name are ready". Only an optimization: a lost hint delays a job until the
+// next poll, it never loses one.
+
+/** At most one message per name in this window; the messages in it make one more at its end */
+const THROTTLE_MS = 100;
+/** The message after a release of mutex keys: every process probes all its names */
+export const ANY_NAME = '*';
+
+/**
+ * @typedef {object} HintChannel
+ * @property {(name: string) => void} publish Fire-and-forget: never throws, never waits.
+ * @property {(onMessage: (name: string) => void) => Promise<void>} listen
+ * @property {() => Promise<void>} stop Stops listening; drops the pending messages.
+ */
+
+/**
+ * @param {object} params
+ * @param {import('./types.js').Hint | null} params.hint
+ * @param {string} params.channel
+ * @param {(operation: string, error: unknown) => void} params.reportError
+ * @returns {HintChannel}
+ */
+export function createHintChannel({ hint, channel, reportError }) {
+    /** @type {Map<string, { timer: NodeJS.Timeout, again: boolean }>} */
+    const windows = new Map();
+    /** @type {(() => Promise<void>) | null} */
+    let unsubscribe = null;
+    let stopped = false;
+
+    /** @param {string} name */
+    function send(name) {
+        if (!hint || stopped) {
+            return;
+        }
+
+        const { publish } = hint;
+
+        // Not awaited: the enqueue does not wait for the hint
+        (async () => {
+            try {
+                await publish(channel, name);
+            } catch (error) {
+                reportError('hint_publish', error);
+            }
+        })();
+    }
+
+    /** @param {string} name */
+    function openWindow(name) {
+        const timer = setTimeout(() => {
+            const window = windows.get(name);
+
+            windows.delete(name);
+
+            if (window?.again) {
+                send(name);
+                openWindow(name);
+            }
+        }, THROTTLE_MS);
+
+        timer.unref();
+        windows.set(name, { timer, again: false });
+    }
+
+    return {
+        publish: (name) => {
+            const window = windows.get(name);
+
+            if (window) {
+                window.again = true;
+
+                return;
+            }
+
+            send(name);
+            openWindow(name);
+        },
+        listen: async (onMessage) => {
+            if (!hint) {
+                return;
+            }
+
+            unsubscribe = await hint.subscribe(channel, (message) => {
+                try {
+                    onMessage(message);
+                } catch (error) {
+                    reportError('hint_message', error);
+                }
+            });
+        },
+        stop: async () => {
+            stopped = true;
+
+            for (const { timer } of windows.values()) {
+                clearTimeout(timer);
+            }
+
+            windows.clear();
+
+            try {
+                await unsubscribe?.();
+            } catch (error) {
+                reportError('hint_unsubscribe', error);
+            }
+        },
+    };
+}

package/src/index.js

@@ -1,2 +1,64 @@
-/* oxlint-disable unicorn/no-empty-file -- placeholder until the first module */
-// Public API of sqlane.
+// The public API of sqlane. See docs/design.md.
+
+export { createSqlane } from './create_sqlane.js';
+export { defineJob } from './define_job.js';
+export {
+    ClosedError,
+    ConfigError,
+    DrainTimeoutError,
+    FatalJobError,
+    InvalidMutexKeyError,
+    InvalidOptionsError,
+    JobTimeoutError,
+    LeaseLostError,
+    MigrationLockTimeoutError,
+    MissingPayloadError,
+    PayloadTooLargeError,
+    RetryJobError,
+    ShutdownError,
+    SqlaneError,
+    UnknownJobError,
+} from './errors.js';
+export { getMigrations } from './migrations.js';
+
+/**
+ * @typedef {import('./types.js').Adapter<C>} Adapter
+ * @template C
+ */
+/**
+ * @typedef {import('./types.js').Sqlane<C, L>} Sqlane
+ * @template C
+ * @template {string} L
+ */
+/**
+ * @typedef {import('./types.js').SqlaneOptions<C, L>} SqlaneOptions
+ * @template C
+ * @template {string} L
+ */
+/**
+ * @typedef {import('./types.js').Jsonable<T>} Jsonable
+ * @template T
+ */
+/** @typedef {import('./types.js').Backoff} Backoff */
+/** @typedef {import('./types.js').CloseOptions} CloseOptions */
+/** @typedef {import('./types.js').DrainOptions} DrainOptions */
+/** @typedef {import('./types.js').DrainResult} DrainResult */
+/** @typedef {import('./types.js').Duration} Duration */
+/** @typedef {import('./types.js').EnqueueOptions} EnqueueOptions */
+/** @typedef {import('./types.js').EnqueueResult} EnqueueResult */
+/** @typedef {import('./types.js').Executor} Executor */
+/** @typedef {import('./types.js').FailedJobPayload} FailedJobPayload */
+/** @typedef {import('./types.js').Handler} Handler */
+/** @typedef {import('./types.js').HandlerContext} HandlerContext */
+/** @typedef {import('./types.js').Hint} Hint */
+/** @typedef {import('./types.js').JobDefaults} JobDefaults */
+/** @typedef {import('./types.js').JobDefinition} JobDefinition */
+/** @typedef {import('./types.js').JobInfo} JobInfo */
+/** @typedef {import('./types.js').JobOptions} JobOptions */
+/** @typedef {import('./types.js').JobStats} JobStats */
+/** @typedef {import('./types.js').LaneOptions} LaneOptions */
+/** @typedef {import('./types.js').Logger} Logger */
+/** @typedef {import('./types.js').MigrateOptions} MigrateOptions */
+/** @typedef {import('./types.js').Migration} Migration */
+/** @typedef {import('./types.js').Retention} Retention */
+/** @typedef {import('./types.js').SqlaneEvent} SqlaneEvent */