@sqlane/core 0.0.1 → 0.1.0

package/README.md

@@ -4,15 +4,53 @@
 
 Transactional SQL job queue for Node.js, backed by MySQL or PostgreSQL.
 
-> Work in progress: this version is a name placeholder, with no code yet. See the [design](docs/design.md).
-
-## Goals
+> Early version (0.0.x): the API can still change. See the [design](docs/design.md).
 
 - Enqueue a job in your business transaction: no commit, no job.
-- Many queues at low cost: idle queues add no load to the database.
+- Many job names at low cost: idle names add no load to the database.
 - Low latency: a job starts in milliseconds after the commit.
-- MySQL and PostgreSQL.
-- MIT license.
+- Mutex keys: jobs with a common key never run at the same time, in the whole cluster.
+- MySQL 8.4+ and PostgreSQL 15+, with `mysql2` or `pg`. Node.js 22+. MIT license.
+
+## Usage
+
+```sh
+npm install @sqlane/core mysql2   # or pg
+```
+
+```js
+import mysql from 'mysql2/promise';
+import { createSqlane, defineJob } from '@sqlane/core';
+import { mysql2Adapter } from '@sqlane/core/mysql2'; // or pgAdapter from '@sqlane/core/pg'
+
+// Shared by producers and workers
+export const buildReport = defineJob({
+    name: 'build_report',
+    maxAttempts: 5,
+    backoff: { type: 'exponential', delay: '5s', maxDelay: '1h' },
+});
+
+const queuePool = mysql.createPool({ uri: process.env.DATABASE_URL, connectionLimit: 3 });
+const sqlane = createSqlane({ adapter: mysql2Adapter(queuePool), jobs: [buildReport] });
+
+await sqlane.migrate(); // or getMigrations({ dialect: 'mysql' }) for your own migration tool
+
+// Producer: with the connection of your transaction
+await connection.beginTransaction();
+await connection.query('INSERT INTO reports (id) VALUES (?)', [reportId]);
+await sqlane.enqueue(connection, buildReport, { reportId }, { jobId: reportId });
+await connection.commit();
+
+// Worker
+sqlane.handle(buildReport, async ({ payload, signal }) => {
+    await build(payload.reportId, { signal });
+}, { concurrency: 2, timeout: '5m' });
+await sqlane.start();
+
+process.once('SIGTERM', () => sqlane.close({ deadline: '30s' }));
+```
+
+In tests, run the due jobs without `start()`: `await sqlane.drain()`. More in [`examples/`](examples).
 
 ## Development
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sqlane/core",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Transactional SQL job queue for Node.js (MySQL, PostgreSQL)",
   "keywords": [
     "background-jobs",
@@ -26,11 +26,17 @@
   ],
   "type": "module",
   "exports": {
-    ".": "./src/index.js"
+    ".": "./src/index.js",
+    "./mysql2": "./src/adapters/mysql2.js",
+    "./pg": "./src/adapters/pg.js"
   },
   "publishConfig": {
     "access": "public"
   },
+  "dependencies": {
+    "parse-duration": "^2.1.11",
+    "ulid": "^3.0.2"
+  },
   "devDependencies": {
     "@changesets/changelog-github": "1.0.1",
     "@changesets/cli": "3.0.3",
@@ -49,6 +55,18 @@
     "typescript": "7.0.2",
     "vitest": "5.0.5"
   },
+  "peerDependencies": {
+    "mysql2": "^3.11.0",
+    "pg": "^8.13.0"
+  },
+  "peerDependenciesMeta": {
+    "mysql2": {
+      "optional": true
+    },
+    "pg": {
+      "optional": true
+    }
+  },
   "engines": {
     "node": ">=22.12.0"
   },

package/src/adapters/mysql2.js

@@ -0,0 +1,96 @@
+/**
+ * @typedef {import('mysql2/promise').Pool
+ *     | import('mysql2/promise').PoolConnection
+ *     | import('mysql2/promise').Connection} Mysql2Connection
+ */
+
+/**
+ * @param {Mysql2Connection} connection
+ * @returns {import('../types.js').Executor}
+ */
+function toExecutor(connection) {
+    return {
+        query: async (sql, params) => {
+            // The three types have the same query(); TS cannot call a method of a union of overloads.
+            const queryable = /** @type {import('mysql2/promise').Connection} */ (connection);
+            const [rows] = await queryable.query(sql, [...params]);
+
+            return Array.isArray(rows) ? /** @type {Record<string, unknown>[]} */ (rows) : [];
+        },
+        run: async (sql, params) => {
+            const queryable = /** @type {import('mysql2/promise').Connection} */ (connection);
+            const [result] = await queryable.query(sql, [...params]);
+
+            return /** @type {import('mysql2/promise').ResultSetHeader} */ (result).affectedRows;
+        },
+    };
+}
+
+/**
+ * @param {Mysql2Connection} connection
+ * @returns {connection is import('mysql2/promise').Pool}
+ */
+function isPool(connection) {
+    return typeof (/** @type {{ getConnection?: unknown }} */ (connection).getConnection) === 'function';
+}
+
+/**
+ * Runs func in a transaction of its own, on a connection of the pool.
+ * @template T
+ * @param {import('mysql2/promise').Pool} pool
+ * @param {(executor: import('../types.js').Executor) => Promise<T>} func
+ * @param {boolean} [readCommitted] Default: false: the isolation level of the session.
+ * @returns {Promise<T>}
+ */
+async function inOwnTransaction(pool, func, readCommitted = false) {
+    const connection = await pool.getConnection();
+
+    try {
+        if (readCommitted) {
+            // For the next transaction only
+            await connection.query('SET TRANSACTION ISOLATION LEVEL READ COMMITTED');
+        }
+
+        await connection.beginTransaction();
+
+        try {
+            const result = await func(toExecutor(connection));
+
+            await connection.commit();
+
+            return result;
+        } catch (error) {
+            await connection.rollback();
+            throw error;
+        }
+    } finally {
+        connection.release();
+    }
+}
+
+/**
+ * The adapter for mysql2 (its promise API). `enqueue` then takes a `Connection`, a `PoolConnection` (in a
+ * transaction) or a `Pool`.
+ * @param {import('mysql2/promise').Pool} pool The queue's own pool: claims, leases, results.
+ * @returns {import('../types.js').Adapter<Mysql2Connection>}
+ */
+export function mysql2Adapter(pool) {
+    return {
+        dialect: 'mysql',
+        wrap: toExecutor,
+        queue: toExecutor(pool),
+        withConnection: async (func) => {
+            const connection = await pool.getConnection();
+
+            try {
+                return await func(toExecutor(connection));
+            } finally {
+                connection.release();
+            }
+        },
+        queueTransaction: func => inOwnTransaction(pool, func, true),
+        inTransaction: (connection, func) => (isPool(connection)
+            ? inOwnTransaction(connection, func)
+            : func(toExecutor(connection))),
+    };
+}

package/src/adapters/pg.js

@@ -0,0 +1,164 @@
+/** @typedef {import('pg').Pool | import('pg').PoolClient | import('pg').Client} PgConnection */
+
+/**
+ * @typedef {object} PgAdapterOptions
+ * @property {boolean} [notify] Hints by LISTEN/NOTIFY: `pg_notify` in the enqueue transaction, delivered on commit.
+ *     Default: false.
+ */
+
+/**
+ * @param {PgConnection} connection
+ * @returns {import('../types.js').Executor}
+ */
+function toExecutor(connection) {
+    return {
+        query: async (sql, params) => {
+            const result = await connection.query(sql, [...params]);
+
+            return result.rows;
+        },
+        run: async (sql, params) => {
+            const result = await connection.query(sql, [...params]);
+
+            return result.rowCount ?? 0;
+        },
+    };
+}
+
+/**
+ * @param {PgConnection} connection
+ * @returns {connection is import('pg').Pool}
+ */
+function isPool(connection) {
+    return typeof (/** @type {{ totalCount?: unknown }} */ (connection).totalCount) === 'number';
+}
+
+/**
+ * Runs func in a transaction of its own, on a client of the pool.
+ * @template T
+ * @param {import('pg').Pool} pool
+ * @param {(executor: import('../types.js').Executor) => Promise<T>} func
+ * @param {boolean} [readCommitted] Default: false: the isolation level of the session.
+ * @returns {Promise<T>}
+ */
+async function inOwnTransaction(pool, func, readCommitted = false) {
+    const client = await pool.connect();
+
+    try {
+        await client.query(readCommitted ? 'BEGIN ISOLATION LEVEL READ COMMITTED' : 'BEGIN');
+
+        try {
+            const result = await func(toExecutor(client));
+
+            await client.query('COMMIT');
+
+            return result;
+        } catch (error) {
+            await client.query('ROLLBACK');
+            throw error;
+        }
+    } finally {
+        client.release();
+    }
+}
+
+/**
+ * LISTEN on a client of its own, taken from the pool; on an error of the client, listens again on a new one.
+ * @param {import('pg').Pool} pool
+ * @param {string} channel A safe identifier: `<tablePrefix>ready`.
+ * @param {(message: string) => void} onMessage
+ * @returns {Promise<() => Promise<void>>} Unlisten.
+ */
+async function listen(pool, channel, onMessage) {
+    /** @type {import('pg').PoolClient | null} */
+    let client = null;
+    let stopped = false;
+
+    /** @param {import('pg').Notification} notification */
+    const onNotification = (notification) => {
+        if (notification.channel === channel) {
+            onMessage(notification.payload ?? '');
+        }
+    };
+
+    /** @returns {Promise<void>} */
+    const connect = async () => {
+        const next = await pool.connect();
+
+        next.on('notification', onNotification);
+        next.once('error', (error) => {
+            next.release(error);
+
+            if (client === next) {
+                client = null;
+            }
+
+            reconnectLater();
+        });
+        await next.query(`LISTEN "${channel}"`);
+        client = next;
+    };
+
+    /** Listens again in 1 s, until it works or the listener stops. */
+    function reconnectLater() {
+        if (stopped) {
+            return;
+        }
+
+        setTimeout(async () => {
+            try {
+                await connect();
+            } catch {
+                reconnectLater();
+            }
+        }, 1000).unref();
+    }
+
+    await connect();
+
+    return async () => {
+        stopped = true;
+
+        const current = client;
+
+        client = null;
+
+        if (current) {
+            try {
+                await current.query(`UNLISTEN "${channel}"`);
+            } finally {
+                current.removeListener('notification', onNotification);
+                current.release();
+            }
+        }
+    };
+}
+
+/**
+ * The adapter for pg. `enqueue` then takes a `Client`, a `PoolClient` (in a transaction) or a `Pool`.
+ * @param {import('pg').Pool} pool The queue's own pool: claims, leases, results.
+ * @param {PgAdapterOptions} [options]
+ * @returns {import('../types.js').Adapter<PgConnection>}
+ */
+export function pgAdapter(pool, options = {}) {
+    return {
+        dialect: 'postgres',
+        wrap: toExecutor,
+        queue: toExecutor(pool),
+        withConnection: async (func) => {
+            const client = await pool.connect();
+
+            try {
+                return await func(toExecutor(client));
+            } finally {
+                client.release();
+            }
+        },
+        queueTransaction: func => inOwnTransaction(pool, func, true),
+        inTransaction: (connection, func) => (isPool(connection)
+            ? inOwnTransaction(connection, func)
+            : func(toExecutor(connection))),
+        notify: options.notify ?? false,
+        listen: (channel, onMessage) => listen(pool, channel, onMessage),
+    };
+}

package/src/backoff.js

@@ -0,0 +1,24 @@
+import { parseDuration } from './duration.js';
+
+/**
+ * The delay before the next attempt.
+ * @param {import('./types.js').Backoff} backoff Checked.
+ * @param {number} attempt The attempt that failed: 1 for the first.
+ * @param {() => number} [random] For tests. Default: `Math.random`.
+ * @returns {number} Milliseconds.
+ */
+export function getBackoffMs(backoff, attempt, random = Math.random) {
+    const delayMs = /** @type {number} */ (parseDuration(backoff.delay));
+    const jitter = backoff.jitter ?? 0.2;
+    let baseMs = delayMs;
+
+    if (backoff.type === 'exponential') {
+        const maxDelayMs = backoff.maxDelay === undefined
+            ? Infinity
+            : /** @type {number} */ (parseDuration(backoff.maxDelay));
+
+        baseMs = Math.min(delayMs * 2 ** (attempt - 1), maxDelayMs);
+    }
+
+    return Math.round(baseMs * (1 - jitter * random()));
+}

package/src/create_sqlane.js

@@ -0,0 +1,210 @@
+import { resolveDefaults } from './defaults.js';
+import { createEnqueuer } from './enqueue.js';
+import { createHintChannel } from './hint_channel.js';
+import { ClosedError, ConfigError } from './errors.js';
+import { runMigrations } from './migrations.js';
+import { DEFAULT_MAX_PAYLOAD_BYTES } from './payload.js';
+import { createRegistry } from './registry.js';
+import { createRunner } from './runner.js';
+import { createStore } from './store/store.js';
+import { checkDuration, checkInteger, checkObject, checkTablePrefix, isPlainObject } from './validate.js';
+
+const OPTION_NAMES = [
+    'adapter', 'jobs', 'concurrency', 'lanes', 'defaults', 'unknownJobs', 'hint', 'logger', 'onEvent', 'tablePrefix',
+    'pollInterval', 'maxPayloadBytes', 'cleanup', 'timings',
+];
+
+/**
+ * @param {unknown} jobs
+ * @param {import('./validate.js').Scope} scope
+ * @returns {Map<string, import('./types.js').JobDefinition>} The definitions by name.
+ */
+function indexJobs(jobs, scope) {
+    if (!Array.isArray(jobs)) {
+        throw new ConfigError(`${scope.where}: "jobs" must be an array of job definitions`);
+    }
+
+    /** @type {Map<string, import('./types.js').JobDefinition>} */
+    const byName = new Map();
+
+    for (const job of jobs) {
+        if (!isPlainObject(job) || typeof job.name !== 'string') {
+            throw new ConfigError(`${scope.where}: "jobs" must be an array of job definitions`);
+        }
+
+        const known = byName.get(job.name);
+
+        if (known && known !== job) {
+            throw new ConfigError(`${scope.where}: two different definitions of "${job.name}" in "jobs"`);
+        }
+
+        byName.set(job.name, /** @type {import('./types.js').JobDefinition} */ (job));
+    }
+
+    return byName;
+}
+
+/**
+ * The hint of an instance: the `hint` option, or the LISTEN/NOTIFY of a pg adapter with `notify`.
+ * @param {import('./types.js').Adapter<any>} adapter
+ * @param {unknown} hint
+ * @returns {import('./types.js').Hint | null}
+ */
+function resolveHint(adapter, hint) {
+    if (hint !== undefined && hint !== null) {
+        if (!isPlainObject(hint) || typeof hint.publish !== 'function' || typeof hint.subscribe !== 'function') {
+            throw new ConfigError('createSqlane: "hint" must be { publish(channel, message), subscribe(channel, onMessage) }');
+        }
+
+        if (adapter.notify) {
+            throw new ConfigError('createSqlane: "hint" or pgAdapter(pool, { notify: true }), not both');
+        }
+
+        return /** @type {import('./types.js').Hint} */ (hint);
+    }
+
+    if (!adapter.notify || !adapter.listen) {
+        return null;
+    }
+
+    const { listen } = adapter;
+
+    return {
+        publish: async (channel, message) => {
+            await adapter.queue.query('SELECT pg_notify($1, $2)', [channel, message]);
+        },
+        subscribe: listen,
+    };
+}
+
+/**
+ * Creates a sqlane instance. It connects, migrates and starts nothing.
+ * @template C The connections that `enqueue` accepts, from the adapter.
+ * @template {string} [L=never] The lane names, from `lanes`.
+ * @param {import('./types.js').SqlaneOptions<C, L>} options
+ * @returns {import('./types.js').Sqlane<C, L>}
+ */
+export function createSqlane(options) {
+    const scope = { where: 'createSqlane', ErrorClass: ConfigError };
+
+    checkObject(options, OPTION_NAMES, 'options', scope);
+
+    if (!options.adapter) {
+        throw new ConfigError('createSqlane: "adapter" is required');
+    }
+
+    const {
+        adapter,
+        tablePrefix = 'sqlane_',
+        unknownJobs = 'allow',
+        maxPayloadBytes = DEFAULT_MAX_PAYLOAD_BYTES,
+    } = options;
+
+    checkTablePrefix(tablePrefix, scope);
+    checkInteger(maxPayloadBytes, '"maxPayloadBytes"', scope);
+
+    if (unknownJobs !== 'allow' && unknownJobs !== 'reject') {
+        throw new ConfigError(`createSqlane: "unknownJobs" must be 'allow' or 'reject'`);
+    }
+
+    const { concurrency = 10, cleanup = true } = options;
+
+    checkInteger(concurrency, '"concurrency"', scope);
+
+    if (typeof cleanup !== 'boolean') {
+        throw new ConfigError('createSqlane: "cleanup" must be a boolean');
+    }
+
+    const pollIntervalMs = checkDuration(options.pollInterval ?? '5s', '"pollInterval"', scope, true);
+    const jobsByName = indexJobs(options.jobs ?? [], scope);
+    const defaults = resolveDefaults(options.defaults);
+    const registry = createRegistry({
+        jobsByName,
+        lanes: options.lanes,
+        concurrency,
+        defaultTimeoutMs: defaults.timeoutMs,
+    });
+    const hint = resolveHint(adapter, options.hint);
+    const hintChannel = createHintChannel({
+        hint,
+        channel: `${tablePrefix}ready`,
+        reportError: (operation, error) => {
+            options.logger?.warn({ error, operation }, `sqlane: ${operation} failed`);
+            options.onEvent?.({ type: 'error', operation, error });
+        },
+    });
+    // NOTIFY in the enqueue transaction: no publish after the enqueue
+    const notifyChannel = adapter.notify ? `${tablePrefix}ready` : null;
+    // Undocumented, for the tests of sqlane
+    const testTimings = /** @type {{ timings?: object }} */ (options).timings;
+    const store = createStore(adapter, tablePrefix);
+    const runner = createRunner({
+        store,
+        registry,
+        concurrency,
+        defaults,
+        cleanup,
+        logger: options.logger,
+        onEvent: options.onEvent,
+        hintChannel,
+        timings: { pollIntervalMs, ...testTimings },
+    });
+    const enqueuer = createEnqueuer({
+        adapter,
+        tablePrefix,
+        findDefinition: registry.findDefinition,
+        unknownJobs,
+        defaults,
+        maxPayloadBytes,
+        notifyChannel,
+    });
+    /** @param {string} name */
+    const onCreated = (name) => {
+        runner.onEnqueued([name]);
+
+        // A process sends a hint only for the jobs that it cannot start itself
+        if (!notifyChannel && !runner.canStartNow(name)) {
+            hintChannel.publish(name);
+        }
+    };
+
+    return {
+        enqueue: async (connection, job, payload, enqueueOptions) => {
+            const result = await enqueuer.enqueue(connection, job, payload, enqueueOptions);
+
+            if (result.created) {
+                onCreated(typeof job === 'string' ? job : job.name);
+            }
+
+            return result;
+        },
+        enqueueMany: async (connection, job, items) => {
+            const results = await enqueuer.enqueueMany(connection, job, items);
+
+            if (results.some(result => result.created)) {
+                onCreated(typeof job === 'string' ? job : job.name);
+            }
+
+            return results;
+        },
+        handle: (job, handler, handleOptions) => {
+            if (runner.isClosed()) {
+                throw new ClosedError('handle() after close()');
+            }
+
+            if (runner.isStarted()) {
+                throw new ConfigError('handle() after start(): register all the handlers before start()');
+            }
+
+            registry.handle(job, handler, handleOptions);
+        },
+        start: runner.start,
+        drain: runner.drain,
+        close: runner.close,
+        migrate: async migrateOptions => ({
+            applied: await runMigrations(adapter, tablePrefix, migrateOptions),
+        }),
+        getStats: () => store.getStats(),
+        wake: name => runner.wake(name),
+    };
+}

package/src/defaults.js

@@ -0,0 +1,38 @@
+import { parseDuration } from './duration.js';
+import { ConfigError } from './errors.js';
+import { checkJobFields, checkObject } from './validate.js';
+
+/**
+ * The job defaults of an instance, merged with the built-in ones and checked.
+ * @typedef {object} ResolvedDefaults
+ * @property {number} maxAttempts
+ * @property {number} maxStalls
+ * @property {import('./types.js').Backoff} backoff
+ * @property {{ completedMs: number, failedMs: number }} retention
+ * @property {number | null} timeoutMs Null: no timeout.
+ */
+
+/** @type {Required<import('./types.js').JobDefaults>['backoff']} */
+const DEFAULT_BACKOFF = Object.freeze({ type: 'exponential', delay: '5s', maxDelay: '1h', jitter: 0.2 });
+
+/**
+ * @param {import('./types.js').JobDefaults} [defaults]
+ * @returns {ResolvedDefaults}
+ */
+export function resolveDefaults(defaults = {}) {
+    const scope = { where: 'createSqlane: "defaults"', ErrorClass: ConfigError };
+
+    checkObject(defaults, ['maxAttempts', 'maxStalls', 'backoff', 'retention', 'timeout'], 'defaults', scope);
+    checkJobFields(defaults, scope);
+
+    return {
+        maxAttempts: defaults.maxAttempts ?? 3,
+        maxStalls: defaults.maxStalls ?? 2,
+        backoff: defaults.backoff ?? DEFAULT_BACKOFF,
+        retention: {
+            completedMs: /** @type {number} */ (parseDuration(defaults.retention?.completed ?? '1h')),
+            failedMs: /** @type {number} */ (parseDuration(defaults.retention?.failed ?? '30d')),
+        },
+        timeoutMs: defaults.timeout === undefined ? null : parseDuration(defaults.timeout),
+    };
+}

package/src/define_job.js

@@ -0,0 +1,40 @@
+import { ConfigError } from './errors.js';
+import { checkFailureJob, checkJobFields, checkObject, JOB_NAME_REGEXP } from './validate.js';
+
+const JOB_OPTION_NAMES = [
+    'name', 'maxAttempts', 'maxStalls', 'backoff', 'retention', 'timeout', 'mutexKeys', 'failureJob',
+];
+
+/**
+ * Defines a job name and its options. Producers and workers import the definition.
+ * @param {import('./types.js').JobOptions} options
+ * @returns {import('./types.js').JobDefinition} Frozen.
+ */
+export function defineJob(options) {
+    const name = typeof options?.name === 'string' ? options.name : '';
+    const where = `defineJob(${JSON.stringify(name)})`;
+
+    const scope = { where, ErrorClass: ConfigError };
+
+    checkObject(options, JOB_OPTION_NAMES, 'options', scope);
+
+    if (!JOB_NAME_REGEXP.test(name)) {
+        throw new ConfigError(`${where}: "name" must match ${JOB_NAME_REGEXP}`);
+    }
+
+    checkJobFields(options, scope);
+
+    if (options.mutexKeys !== undefined && typeof options.mutexKeys !== 'function') {
+        throw new ConfigError(`${where}: "mutexKeys" must be a function of the payload`);
+    }
+
+    if (options.failureJob !== undefined) {
+        checkFailureJob(options.failureJob, name, scope);
+    }
+
+    return Object.freeze({
+        ...options,
+        ...(options.backoff && { backoff: Object.freeze({ ...options.backoff }) }),
+        ...(options.retention && { retention: Object.freeze({ ...options.retention }) }),
+    });
+}