@adonis-agora/durable 1.0.0-rc.0 → 1.0.0-rc.10

package/dist/src/stores/lucid.js

@@ -1,4 +1,6 @@
-import { SETTLEABLE_RUN_STATUSES } from '../interfaces.js';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { RunAlreadyExistsError } from '../errors.js';
+import { RECOVERABLE_RUN_STATUSES, SETTLEABLE_RUN_STATUSES } from '../interfaces.js';
 import { namespaceFilterValues, normalizeNamespace } from '../namespaces.js';
 import { mergeRunFacetRows } from '../run-facets.js';
 import { runPageWindow } from '../run-pagination.js';
@@ -51,19 +53,59 @@ export class LucidStateStore {
     }
     /**
      * The Postgres hot path (see `lucid-pg-hot-path.ts`): the statements a turn and a step result run
-     * on every step, as constant SQL. `undefined` on any other dialect, or with Lucid's `debug` on.
-     * Resolved per call, like {@link client}: under a global transaction it is the transaction's.
+     * on every step, as constant SQL. `undefined` on any other dialect, with Lucid's `debug` on, or
+     * under a (global) transaction. Resolved per call, like {@link client}.
      */
     hot() {
+        // Not on a transaction (an app's global one): see `LucidPgHotPath.for`.
         return LucidPgHotPath.for(this.client(), {
             preparedStatements: this.options.preparedStatements,
         });
     }
+    /**
+     * The hot reads with explicit column lists: each table's base columns plus the later-wave ones it
+     * has, probed once (an older schema without them keeps working) and again after this store's own
+     * {@link ensureSchema}. A column another process adds later is not read until the next probe, and a
+     * statement already prepared keeps its shape.
+     */
+    hotReadStatements;
+    hotReads(hot) {
+        this.hotReadStatements ??= hot
+            .rows(COLUMNS_PRESENT_SQL, [
+            [DURABLE_TABLES.runs, DURABLE_TABLES.checkpoints],
+            [...RUN_OPTIONAL_COLUMNS, ...CHECKPOINT_OPTIONAL_COLUMNS],
+        ])
+            .then((rows) => {
+            const present = new Set(rows.map((r) => `${String(r.table_name)}.${String(r.column_name)}`));
+            const has = (table) => (c) => present.has(`${table}.${c}`);
+            const runCols = [
+                ...RUN_BASE_COLUMNS,
+                ...RUN_OPTIONAL_COLUMNS.filter(has(DURABLE_TABLES.runs)),
+            ];
+            const checkpointCols = [
+                ...CHECKPOINT_BASE_COLUMNS,
+                ...CHECKPOINT_OPTIONAL_COLUMNS.filter(has(DURABLE_TABLES.checkpoints)),
+            ];
+            return hotReadSql(runCols.map(quote).join(', '), checkpointCols.map(quote).join(', '));
+        })
+            .catch((error) => {
+            this.hotReadStatements = undefined; // a failed probe is retried, never cached
+            throw error;
+        });
+        return this.hotReadStatements;
+    }
     /** Idempotently provision the durable tables (called on boot when `autoSchema` is on). */
     async ensureSchema() {
         await createDurableTables(this.db, this.connectionName, this.options.collation !== undefined ? { collation: this.options.collation } : {});
+        // The tables may have gained columns: the next hot read probes them again.
+        this.hotReadStatements = undefined;
     }
     // --- runs ---------------------------------------------------------------
+    /**
+     * Insert the run, or reject with {@link RunAlreadyExistsError} when its id is taken. The insert
+     * writes nothing on a conflict instead of failing, so a run that already exists never aborts a
+     * surrounding transaction (an app's test `wrapInGlobalTransaction`, on Postgres).
+     */
     async createRun(run) {
         if (normalizeAttributeRows(run.id, run.searchAttributes).length === 0) {
             // Nothing to index: the insert, then clear any attribute rows a deleted run of this id left,
@@ -72,17 +114,23 @@ export class LucidStateStore {
             if (hot) {
                 const row = runToRow(run);
                 const keys = Object.keys(row);
-                await hot.run(insertSql(DURABLE_TABLES.runs, keys), keys.map((k) => row[k]));
+                const inserted = await hot.rows(insertRunIfAbsentSql(keys), keys.map((k) => row[k]));
+                if (inserted.length === 0)
+                    throw new RunAlreadyExistsError(run.id);
                 await hot.run(DELETE_ATTRIBUTES_SQL, [run.id]);
                 return;
             }
             const client = this.client();
-            await client.table(DURABLE_TABLES.runs).insert(runToRow(run));
+            if (!(await insertIfAbsent(client, DURABLE_TABLES.runs, runToRow(run), ['id']))) {
+                throw new RunAlreadyExistsError(run.id);
+            }
             await client.from(DURABLE_TABLES.attributes).where('run_id', run.id).delete();
             return;
         }
         await this.client().transaction(async (trx) => {
-            await trx.table(DURABLE_TABLES.runs).insert(runToRow(run));
+            if (!(await insertIfAbsent(trx, DURABLE_TABLES.runs, runToRow(run), ['id']))) {
+                throw new RunAlreadyExistsError(run.id);
+            }
             await this.reindexAttributes(trx, run.id, run.searchAttributes);
         });
     }
@@ -277,9 +325,10 @@ export class LucidStateStore {
     async getRun(runId) {
         const hot = this.hot();
         if (hot) {
+            const sql = await this.hotReads(hot);
             const rows = this.scopeNamespace === undefined
-                ? await hot.rows(GET_RUN_SQL, [runId])
-                : await hot.rows(GET_RUN_SCOPED_SQL, [runId, namespaceFilterValues(this.scopeNamespace)]);
+                ? await hot.rows(sql.get, [runId])
+                : await hot.rows(sql.getScoped, [runId, namespaceFilterValues(this.scopeNamespace)]);
             return rows[0] ? rowToRun(rows[0]) : null;
         }
         const row = await this.fenced(this.client().from(DURABLE_TABLES.runs).where('id', runId)).first();
@@ -323,7 +372,7 @@ export class LucidStateStore {
     async getCheckpoint(runId, seq) {
         const hot = this.hot();
         if (hot) {
-            const rows = await hot.rows(GET_CHECKPOINT_SQL, [runId, seq]);
+            const rows = await hot.rows((await this.hotReads(hot)).getCheckpoint, [runId, seq]);
             return rows[0] ? rowToCheckpoint(rows[0]) : null;
         }
         const row = await this.client()
@@ -587,27 +636,7 @@ export class LucidStateStore {
             const values = keys.map((k) => row[k] ?? null);
             return (await hot.rows(insertCheckpointIfAbsentSql(keys), values)).length > 0;
         }
-        if (client.dialect.name === 'postgres') {
-            const inserted = (await client
-                .insertQuery()
-                .table(DURABLE_TABLES.checkpoints)
-                .knexQuery.insert(row)
-                .onConflict(['run_id', 'seq'])
-                .ignore()
-                .returning('seq'));
-            return inserted.length > 0;
-        }
-        // MySQL and SQLite: a plain INSERT, whose duplicate-key error is the "already there" answer (an
-        // `INSERT IGNORE` would also swallow every other error, and reports no row count through Knex).
-        try {
-            await client.table(DURABLE_TABLES.checkpoints).insert(row);
-            return true;
-        }
-        catch (err) {
-            if (isDuplicateKey(err))
-                return false;
-            throw err;
-        }
+        return insertIfAbsent(client, DURABLE_TABLES.checkpoints, row, ['run_id', 'seq']);
     }
     /** Persist the latest worker heartbeat on the step's checkpoint row (see the interface doc).
      *  A targeted two-column UPDATE — never an upsert: a beat racing the checkpoint's own insert (or
@@ -639,7 +668,9 @@ export class LucidStateStore {
     }
     // --- recovery / dispatch queries ----------------------------------------
     async listIncompleteRuns(namespace) {
-        const q = this.fenced(this.client().from(DURABLE_TABLES.runs).where('status', 'running'));
+        const q = this.fenced(this.client()
+            .from(DURABLE_TABLES.runs)
+            .whereIn('status', [...RECOVERABLE_RUN_STATUSES]));
         if (namespace !== undefined)
             q.whereIn('namespace', namespaceFilterValues(namespace));
         const rows = await q;
@@ -652,7 +683,7 @@ export class LucidStateStore {
         // instance id (a rebooted instance's own leftovers; the engine filters what it is executing).
         const q = this.fenced(this.client()
             .from(DURABLE_TABLES.runs)
-            .where('status', 'running')
+            .whereIn('status', [...RECOVERABLE_RUN_STATUSES])
             .andWhere((sub) => {
             sub.whereNull('locked_until').orWhere('locked_until', '<=', nowMs);
             if (owner !== undefined)
@@ -767,9 +798,10 @@ export class LucidStateStore {
     async tryLockRunAndGet(runId, owner, leaseUntilMs, nowMs) {
         const hot = this.hot();
         if (hot) {
+            const sql = await this.hotReads(hot);
             const rows = this.scopeNamespace === undefined
-                ? await hot.rows(LOCK_AND_GET_SQL, [owner, leaseUntilMs, runId, nowMs])
-                : await hot.rows(LOCK_AND_GET_SCOPED_SQL, [
+                ? await hot.rows(sql.lockAndGet, [owner, leaseUntilMs, runId, nowMs])
+                : await hot.rows(sql.lockAndGetScoped, [
                     owner,
                     leaseUntilMs,
                     runId,
@@ -1568,6 +1600,43 @@ export class LucidStateStore {
         }
     }
 }
+/**
+ * Serialize the store's work on a transaction client. A transaction is ONE connection: two store
+ * calls in flight at once (a background run pickup next to a request's own write, under an app's test
+ * `wrapInGlobalTransaction`) would send overlapping queries down it, which `pg` deprecates (and 9.0
+ * removes), and a statement could land inside a savepoint the other call is rolling back. So while
+ * the store's client is a transaction, each call waits for the previous one on that transaction to
+ * finish. Re-entrant: a store method calling another one in the same call does not wait for itself.
+ * A store on a plain connection is untouched (its pool hands each query its own connection).
+ */
+const transactionTails = new WeakMap();
+const holdingTransaction = new AsyncLocalStorage();
+function serializeOnTransactions(proto) {
+    const AsyncFunction = (async () => { }).constructor;
+    for (const name of Object.getOwnPropertyNames(proto)) {
+        if (name === 'constructor')
+            continue;
+        const descriptor = Object.getOwnPropertyDescriptor(proto, name);
+        const original = descriptor?.value;
+        if (typeof original !== 'function' || !(original instanceof AsyncFunction))
+            continue;
+        const method = original;
+        Object.defineProperty(proto, name, {
+            ...descriptor,
+            value: function (...args) {
+                const client = this.client();
+                if (!client.isTransaction || holdingTransaction.getStore() === client) {
+                    return method.apply(this, args);
+                }
+                const previous = transactionTails.get(client) ?? Promise.resolve();
+                const run = previous.then(() => holdingTransaction.run(client, () => method.apply(this, args)), () => holdingTransaction.run(client, () => method.apply(this, args)));
+                transactionTails.set(client, run.catch(() => undefined));
+                return run;
+            },
+        });
+    }
+}
+serializeOnTransactions(LucidStateStore.prototype);
 /**
  * Escape LIKE metacharacters (`%`, `_`) — and the escape character itself — using `!` as the ESCAPE
  * char. `!` because a backslash inside a SQL string literal is dialect-divergent (MySQL treats it as
@@ -1664,24 +1733,71 @@ function escapeLike(value) {
 const RUNS_T = quote(DURABLE_TABLES.runs);
 const CHECKPOINTS_T = quote(DURABLE_TABLES.checkpoints);
 const CHECKPOINT_LIST = CHECKPOINT_COLUMNS_BUT_INPUT.map(quote).join(', ');
-const GET_RUN_SQL = `select * from ${RUNS_T} where "id" = ? limit 1`;
-const GET_RUN_SCOPED_SQL = `select * from ${RUNS_T} where "id" = ? and "namespace" = any(?) limit 1`;
+/**
+ * The runs columns every schema has. The hot path names its columns instead of `*`: a named prepared
+ * statement's result type is fixed when it is parsed, so a `*` statement fails ("cached plan must
+ * not change result type") once a column is added under it — another pod's `ensureSchema` during a
+ * rolling upgrade.
+ */
+const RUN_BASE_COLUMNS = [
+    'id',
+    'workflow',
+    'workflow_version',
+    'status',
+    'namespace',
+    'input',
+    'output',
+    'error',
+    'wake_at',
+    'locked_by',
+    'locked_until',
+    'recovery_attempts',
+    'tags',
+    'search_attributes',
+    'priority',
+    'created_at',
+    'updated_at',
+];
+/** Runs columns that arrived in later schema waves: read only when the table has them. */
+const RUN_OPTIONAL_COLUMNS = ['origin', 'awaiting_decision_task_id', 'wake_requested_at'];
+/** Checkpoint columns that arrived in later schema waves: read only when the table has them. */
+const CHECKPOINT_OPTIONAL_COLUMNS = [
+    'queue',
+    'parallel_group',
+    'last_heartbeat_at',
+    'heartbeat_progress',
+];
+const CHECKPOINT_BASE_COLUMNS = [
+    ...CHECKPOINT_COLUMNS_BUT_INPUT.filter((c) => !CHECKPOINT_OPTIONAL_COLUMNS.includes(c)),
+    'input',
+];
+const COLUMNS_PRESENT_SQL = `select "table_name", "column_name" from information_schema.columns where "table_schema" = current_schema() and "table_name" = any(?) and "column_name" = any(?)`;
+const BUFFERED_SIGNAL_LIST = ['id', 'token', 'payload'].map(quote).join(', ');
+const BUFFERED_EVENT_LIST = ['id', 'name', 'payload', 'published_at'].map(quote).join(', ');
+/** {@link HotReads} for the runs column list `cols` and checkpoints column list `checkpointCols`
+ *  (each quoted, comma-joined). */
+function hotReadSql(cols, checkpointCols) {
+    return {
+        getCheckpoint: shapedSql(`checkpoint-get:${checkpointCols}`, () => `select ${checkpointCols} from ${CHECKPOINTS_T} where "run_id" = ? and "seq" = ? limit 1`),
+        get: shapedSql(`run-get:${cols}`, () => `select ${cols} from ${RUNS_T} where "id" = ? limit 1`),
+        getScoped: shapedSql(`run-get-scoped:${cols}`, () => `select ${cols} from ${RUNS_T} where "id" = ? and "namespace" = any(?) limit 1`),
+        lockAndGet: shapedSql(`run-lock:${cols}`, () => `update ${RUNS_T} set "locked_by" = ?, "locked_until" = ? where "id" = ? and ("locked_until" is null or "locked_until" <= ?) returning ${cols}`),
+        lockAndGetScoped: shapedSql(`run-lock-scoped:${cols}`, () => `update ${RUNS_T} set "locked_by" = ?, "locked_until" = ? where "id" = ? and ("locked_until" is null or "locked_until" <= ?) and "namespace" = any(?) returning ${cols}`),
+    };
+}
 const RUN_IN_STATUSES_SQL = `select 1 from ${RUNS_T} where "id" = ? and "status" = any(?) limit 1`;
-const LOCK_AND_GET_SQL = `update ${RUNS_T} set "locked_by" = ?, "locked_until" = ? where "id" = ? and ("locked_until" is null or "locked_until" <= ?) returning *`;
-const LOCK_AND_GET_SCOPED_SQL = `update ${RUNS_T} set "locked_by" = ?, "locked_until" = ? where "id" = ? and ("locked_until" is null or "locked_until" <= ?) and "namespace" = any(?) returning *`;
 const RELEASE_LOCK_SQL = `update ${RUNS_T} set "locked_by" = null, "locked_until" = null where "id" = ?`;
 const RELEASE_OWN_LOCK_SQL = `update ${RUNS_T} set "locked_by" = null, "locked_until" = null where "id" = ? and "locked_by" = ?`;
 const RELEASE_UNLESS_WAKE_SQL = `update ${RUNS_T} set "locked_by" = null, "locked_until" = null where "id" = ? and "locked_by" = ? and "wake_requested_at" is null`;
 const LOCKED_BY_SQL = `select "locked_by" from ${RUNS_T} where "id" = ? limit 1`;
 const RENEW_LOCK_SQL = `update ${RUNS_T} set "locked_until" = ? where "id" = ? and "locked_by" = ?`;
 const DELETE_ATTRIBUTES_SQL = `delete from ${quote(DURABLE_TABLES.attributes)} where "run_id" = ?`;
-const GET_CHECKPOINT_SQL = `select * from ${CHECKPOINTS_T} where "run_id" = ? and "seq" = ? limit 1`;
 const CHECKPOINTS_FOR_SETTLE_SQL = `select ${CHECKPOINT_LIST} from ${CHECKPOINTS_T} where "run_id" = ? and "seq" = any(?)`;
 const REPLAY_CHECKPOINTS_SQL = `select ${CHECKPOINT_LIST} from ${CHECKPOINTS_T} where "run_id" = ? order by "seq" asc`;
 const COUNT_CHECKPOINTS_SQL = `select count(*) as "count" from ${CHECKPOINTS_T} where "run_id" = ? and "seq" >= ? and "seq" <= ? and "status" = any(?)`;
-const OLDEST_BUFFERED_SIGNAL_SQL = `select * from ${quote(DURABLE_TABLES.bufferedSignals)} where "token" = ? order by "id" asc limit 1`;
+const OLDEST_BUFFERED_SIGNAL_SQL = `select ${BUFFERED_SIGNAL_LIST} from ${quote(DURABLE_TABLES.bufferedSignals)} where "token" = ? order by "id" asc limit 1`;
 const DELETE_BUFFERED_SIGNAL_SQL = `delete from ${quote(DURABLE_TABLES.bufferedSignals)} where "id" = ?`;
-const BUFFERED_EVENTS_SQL = `select * from ${quote(DURABLE_TABLES.bufferedEvents)} where "name" = ? order by "published_at" asc limit ?`;
+const BUFFERED_EVENTS_SQL = `select ${BUFFERED_EVENT_LIST} from ${quote(DURABLE_TABLES.bufferedEvents)} where "name" = ? order by "published_at" asc limit ?`;
 /** Statements whose column list is the row's own keys, built once per key set. */
 const shaped = new Map();
 function shapedSql(key, build) {
@@ -1699,6 +1815,10 @@ function insertSql(table, keys) {
         return `insert into ${quote(table)} (${cols}) values (${keys.map(() => '?').join(', ')})`;
     });
 }
+/** {@link insertSql} of a run that writes nothing when its id is taken, returning the id it wrote. */
+function insertRunIfAbsentSql(keys) {
+    return shapedSql(`insert-run-if-absent:${keys.join(',')}`, () => `${insertSql(DURABLE_TABLES.runs, keys)} on conflict ("id") do nothing returning "id"`);
+}
 /** {@link insertSql} of a checkpoint that writes nothing when its position is taken. */
 function insertCheckpointIfAbsentSql(keys) {
     return shapedSql(`insert-if-absent:${keys.join(',')}`, () => `${insertSql(DURABLE_TABLES.checkpoints, keys)} on conflict ("run_id", "seq") do nothing returning "seq"`);
@@ -1758,14 +1878,36 @@ async function settleResultsReturningPg(client, hot, results) {
     }
     return out;
 }
-/** A duplicate-key violation, as MySQL and SQLite report it through Knex. */
-function isDuplicateKey(err) {
-    const e = err;
-    return (e?.code === 'ER_DUP_ENTRY' ||
-        e?.errno === 1062 ||
-        e?.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' ||
-        e?.code === 'SQLITE_CONSTRAINT_UNIQUE' ||
-        e?.code === '23505');
+/**
+ * Insert `row` into `table` unless a row with the same `key` exists, as a statement that never
+ * fails on the conflict, and say whether it wrote. A failing statement would abort a surrounding
+ * Postgres transaction, so the conflict is never detected by catching a duplicate-key error.
+ *
+ * - Postgres and SQLite: `INSERT … ON CONFLICT (key) DO NOTHING RETURNING key`, a row back when it wrote.
+ * - MySQL: `INSERT … ON DUPLICATE KEY UPDATE key = IF(LAST_INSERT_ID(1), key, key)`, which leaves an
+ *   existing row as it is and reports `insertId` 1 for it, 0 for a row it wrote (`table` has no
+ *   AUTO_INCREMENT). The affected-row count cannot tell them apart (mysql2 counts found rows), and
+ *   `INSERT IGNORE` would turn a value the column cannot hold into a silent truncation.
+ */
+async function insertIfAbsent(client, table, row, key) {
+    if (client.dialect.name === 'mysql') {
+        const first = key[0];
+        const knex = client.getWriteClient();
+        const { sql, bindings } = client
+            .insertQuery()
+            .table(table)
+            .knexQuery.insert(row)
+            .onConflict(key)
+            .merge({ [first]: knex.raw('if(last_insert_id(1), ??, ??)', [first, first]) })
+            .toSQL()
+            .toNative();
+        const result = (await client.rawQuery(sql, bindings));
+        const header = (Array.isArray(result) ? result[0] : result);
+        return Number(header?.insertId ?? 0) === 0;
+    }
+    const insert = client.insertQuery().table(table).knexQuery.insert(row).onConflict(key);
+    const inserted = (await insert.ignore().returning(key[0]));
+    return inserted.length > 0;
 }
 function checkpointKey(runId, seq) {
     return `${runId}\u0000${seq}`;