@lenne.tech/nest-server 12.0.0 → 12.1.0

package/src/core/modules/migrate/mongo-state-store.ts

@@ -1,25 +1,25 @@
 import { randomUUID } from 'crypto';
-import { Db, MongoClient } from 'mongodb';
+import type { Db, MongoClient } from 'mongodb';
 import { hostname } from 'os';
 import { promisify } from 'util';
 
+import type { MigrationSet, MigrationStateStore } from './migration-state-store.interface.js';
+import {
+  MIGRATION_LOCK_KEY,
+  MIGRATION_LOCK_STALE_AFTER_MS,
+  migrationMongoLocks,
+} from '../../common/persistence/infrastructure-locks.js';
+import { requireMongoPackages } from '../../common/persistence/mongo/mongo-packages.js';
+
 /**
  * Migration options interface (compatible with migrate package)
  */
 export interface MigrationOptions {
   [key: string]: unknown;
-  stateStore: MongoStateStore;
+  stateStore: MigrationStateStore;
 }
 
-/**
- * Migration set interface (compatible with migrate package)
- */
-export interface MigrationSet {
-  down?: (done?: (err?: Error) => void) => void;
-  lastRun?: string;
-  migrations: Array<{ timestamp?: number; title: string }>;
-  up: (done?: (err?: Error) => void) => void;
-}
+export type { MigrationSet, MigrationStateStore };
 
 /**
  * Options for MongoStateStore configuration
@@ -58,7 +58,7 @@ export interface MongoStateStoreOptions {
  * });
  * ```
  */
-export class MongoStateStore {
+export class MongoStateStore implements MigrationStateStore {
   /** Collection name for storing migration state */
   private readonly collectionName: string;
 
@@ -173,6 +173,12 @@ export async function synchronizedMigration(
 
   const stateStore = opts.stateStore;
 
+  // A store with its own lock (SqlStateStore) serialises the run itself
+  if (!(stateStore instanceof MongoStateStore) && typeof stateStore?.withLock === 'function') {
+    await stateStore.withLock(async () => callback(await stateStore.loadAsync()));
+    return;
+  }
+
   if (!(stateStore instanceof MongoStateStore)) {
     throw new Error('Given `stateStore` is not `MongoStateStore`');
   }
@@ -238,7 +244,11 @@ export async function synchronizedUp(opts: MigrationOptions): Promise<void> {
  * @param fn - Work to execute under the lock
  * @returns Promise with the result of `fn`
  */
-export async function withMigrationLock<T>(stateStore: MongoStateStore, fn: () => Promise<T>): Promise<T> {
+export async function withMigrationLock<T>(stateStore: MigrationStateStore, fn: () => Promise<T>): Promise<T> {
+  if (!(stateStore instanceof MongoStateStore)) {
+    // Another database's store brings its own lock; without one, fn runs unsynchronized
+    return typeof stateStore.withLock === 'function' ? stateStore.withLock(fn) : fn();
+  }
   const lockCollectionName = stateStore.lockCollectionName;
 
   if (!lockCollectionName) {
@@ -262,7 +272,7 @@ export async function withMigrationLock<T>(stateStore: MongoStateStore, fn: () =
  * on migration runtime — a migration running for hours keeps its lock as long as the
  * process lives.
  */
-const LOCK_STALE_AFTER_MS = 60_000;
+const LOCK_STALE_AFTER_MS = MIGRATION_LOCK_STALE_AFTER_MS;
 
 /** Heartbeat interval — comfortably below {@link LOCK_STALE_AFTER_MS} */
 const LOCK_HEARTBEAT_INTERVAL_MS = 15_000;
@@ -271,11 +281,12 @@ const LOCK_HEARTBEAT_INTERVAL_MS = 15_000;
 const LOCK_WAIT_TIMEOUT_MS = 15 * 60_000;
 
 /**
- * Keep refreshing `acquiredAt` while the migration runs.
+ * Keep renewing the lease while the migration runs.
  *
  * This is what makes lock-breaking safe: a lock is only ever taken over when its holder
- * stopped refreshing it, which for a live process cannot happen. Without the heartbeat,
- * breaking a stale lock and a long-running migration would be indistinguishable.
+ * stopped renewing it, which for a live process cannot happen. Without the heartbeat,
+ * breaking a stale lock and a long-running migration would be indistinguishable. The renewal
+ * also refreshes `acquiredAt`, the heartbeat an older replica's waiter reads.
  */
 function startLockHeartbeat(url: string, lockCollectionName: string, token: string): { stop: () => void } {
   // ONE connection for the whole heartbeat, rather than `dbRequest()`'s connect-and-close per
@@ -284,13 +295,14 @@ function startLockHeartbeat(url: string, lockCollectionName: string, token: stri
   // on the first tick (so a heartbeat that never fires costs nothing) and closed by stop().
   let client: MongoClient | undefined;
   let stopped = false;
+  let lostReported = false;
 
   const beat = async (): Promise<void> => {
     if (stopped) {
       return;
     }
     if (!client) {
-      const connected = await MongoClient.connect(url);
+      const connected = await mongoDriver().MongoClient.connect(url);
       if (stopped) {
         // stop() ran while we were connecting — nothing would ever close this one.
         await connected.close();
@@ -298,12 +310,20 @@ function startLockHeartbeat(url: string, lockCollectionName: string, token: stri
       }
       client = connected;
     }
-    // Scoped: once our lock has been broken and re-taken, our heartbeat must not keep the NEW
-    // holder's lock alive on our behalf.
-    await client
-      .db()
-      .collection(lockCollectionName)
-      .updateOne({ lock: 'lock', owner: token }, { $set: { acquiredAt: new Date() } });
+    // Scoped to our token: once our lock has been broken and re-taken, our heartbeat must not
+    // keep the NEW holder's lock alive on our behalf.
+    const renewed = await migrationMongoLocks(client.db().collection(lockCollectionName)).renew(
+      MIGRATION_LOCK_KEY,
+      token,
+      LOCK_STALE_AFTER_MS / 1000,
+    );
+    if (!renewed && !lostReported) {
+      lostReported = true;
+      console.warn(
+        `Migration lock in "${lockCollectionName}" was taken over by another replica while this one was migrating ` +
+          '(missed heartbeats). The migration continues, but no longer exclusively.',
+      );
+    }
   };
 
   const timer = setInterval(() => {
@@ -333,62 +353,34 @@ function startLockHeartbeat(url: string, lockCollectionName: string, token: stri
 /**
  * Acquires a lock in MongoDB to ensure only one migration runs at a time
  *
+ * A lease (`LockStore`) on the 12.0.0 document `{ lock: 'lock', owner, acquiredAt }`, so a replica
+ * still running an older version waits for this one and vice versa (`infrastructure-locks.ts`).
+ *
  * @param url - MongoDB connection URI
  * @param lockCollectionName - Name of the collection to use for locking
  */
 async function acquireLock(url: string, lockCollectionName: string): Promise<string> {
   return dbRequest(url, async (db) => {
     const collection = db.collection(lockCollectionName);
-
-    // Create unique index for atomicity
-    // https://docs.mongodb.com/manual/reference/method/db.collection.update/#use-unique-indexes
-    // https://groups.google.com/forum/#!topic/mongodb-user/-fucdS-7kIU
-    // https://stackoverflow.com/questions/33346175/mongodb-upsert-operation-seems-not-atomic-which-throws-duplicatekeyexception/34784533
-    await collection.createIndex({ lock: 1 }, { unique: true });
-
+    const locks = migrationMongoLocks(collection);
     const token = lockOwnerId();
     const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS;
     let showMessage = true;
 
     for (;;) {
-      // Use updateOne with upsert for atomic lock acquisition (same as original package).
-      // `$setOnInsert` stamps the acquisition time only for the winner, so a loser's
-      // update never refreshes the holder's staleness clock.
-      const result = await collection.updateOne(
-        { lock: 'lock' },
-        { $set: { lock: 'lock' }, $setOnInsert: { acquiredAt: new Date(), owner: token } },
-        { upsert: true },
-      );
-
-      if (result.upsertedCount > 0) {
-        return token;
-      }
-
+      const holder = await collection.findOne({ lock: MIGRATION_LOCK_KEY });
       // The holder may be gone: migrations run on every container boot, so a replica
       // SIGKILLed mid-migration (OOM, node drain, failed deploy) would otherwise leave a
-      // lock nobody holds — and every future boot of every replica would wait on it
-      // forever. Only a lock that stopped heart-beating is broken; see LOCK_STALE_AFTER_MS.
-      const holder = await collection.findOne({ lock: 'lock' });
-      if (holder) {
-        const acquiredAt = holder.acquiredAt instanceof Date ? holder.acquiredAt : undefined;
-        if (!acquiredAt) {
-          // Written by a version that did not stamp the lock. Start its clock now rather
-          // than breaking it immediately — the holder may well be alive.
-          await collection.updateOne(
-            { _id: holder._id, acquiredAt: { $exists: false } },
-            { $set: { acquiredAt: new Date() } },
-          );
-        } else if (Date.now() - acquiredAt.getTime() > LOCK_STALE_AFTER_MS) {
+      // lock nobody holds. The lease breaks only a lock that stopped heart-beating.
+      if (await locks.tryAcquire(MIGRATION_LOCK_KEY, token, LOCK_STALE_AFTER_MS / 1000)) {
+        if (holder) {
+          const lastBeat = holder.acquiredAt instanceof Date ? holder.acquiredAt.toISOString() : 'never';
           console.warn(
-            `Breaking stale migration lock in "${lockCollectionName}" (last heartbeat ${acquiredAt.toISOString()}, ` +
+            `Broke stale migration lock in "${lockCollectionName}" (last heartbeat ${lastBeat}, ` +
               `owner ${holder.owner ?? 'unknown'}) — its holder is gone.`,
           );
-          // Matching on acquiredAt makes the break safe under concurrency: if another
-          // waiter already broke and re-acquired the lock, the timestamp differs and this
-          // deletes nothing.
-          await collection.deleteOne({ _id: holder._id, acquiredAt });
-          continue;
         }
+        return token;
       }
 
       if (Date.now() > deadline) {
@@ -429,7 +421,7 @@ function lockOwnerId(): string {
 async function dbRequest<T>(url: string, callback: (db: Db) => Promise<T> | T): Promise<T> {
   let client: MongoClient | undefined;
   try {
-    client = await MongoClient.connect(url);
+    client = await mongoDriver().MongoClient.connect(url);
     const db = client.db();
     return await callback(db);
   } finally {
@@ -447,5 +439,12 @@ async function releaseLock(url: string, lockCollectionName: string, token: strin
   // Scoped to OUR token. An unscoped delete removes whatever lock exists — so after a missed
   // heartbeat let another replica break and re-take the lock, the original holder's `finally`
   // would delete the NEW holder's lock and let a third replica start a concurrent migration.
-  await dbRequest(url, (db) => db.collection(lockCollectionName).deleteOne({ lock: 'lock', owner: token }));
+  await dbRequest(url, (db) =>
+    migrationMongoLocks(db.collection(lockCollectionName)).release(MIGRATION_LOCK_KEY, token),
+  );
+}
+
+/** The MongoDB driver — a MongoDB migration needs it, a project without MongoDB never reaches this */
+function mongoDriver() {
+  return requireMongoPackages('A MongoDB migration (store or helper)').mongodb;
 }

package/src/core/modules/migrate/sql-migration.helper.ts

@@ -0,0 +1,48 @@
+import type { DatabaseSqlEngine } from '../../common/interfaces/server-options.interface.js';
+import { withMysqlSchemaLock } from '../../common/persistence/sql/mysql/mysql-schema.migrator.js';
+import { SCHEMA_LOCK_SQL } from '../../common/persistence/sql/postgres/postgres-schema.migrator.js';
+import { createSqlExecutor } from '../../common/persistence/sql/sql-executor.factory.js';
+
+/**
+ * Run the reviewed statements of a SQL migration (`migrate create <name> --sql`) as the table OWNER, under the lock the
+ * boot's schema migration takes, so a replica booting meanwhile waits and then sees the result. On PostgreSQL in one
+ * transaction: either every statement applies or none does. MySQL and MariaDB commit every DDL statement on its own, so
+ * there a failure leaves the statements before it applied — write such a migration so it can run again.
+ *
+ * The statements are the ones a refused boot suggested (`SchemaMigrationRequiredError`) — after review: a narrowed
+ * type, a new unique index or a rename collision can reject or rewrite stored rows, which is why the framework never
+ * runs them on its own (plan §3.12).
+ */
+export async function runSqlMigration(
+  ownerUrl: string,
+  statements: readonly string[],
+  options: { engine?: DatabaseSqlEngine } = {},
+): Promise<void> {
+  if (!ownerUrl) {
+    throw new Error('runSqlMigration() needs the connection URL of the table owner (database.ownerUrl)');
+  }
+  const executor = await createSqlExecutor({
+    applicationName: 'nest-server-migration',
+    engine: options.engine ?? 'typeorm',
+    maxConnections: 1,
+    url: ownerUrl,
+  });
+  try {
+    if (/^(mysql|mariadb):/i.test(ownerUrl)) {
+      await withMysqlSchemaLock(executor, async (run) => {
+        for (const text of statements) {
+          await run({ params: [], text });
+        }
+      });
+      return;
+    }
+    await executor.transaction(async (tx) => {
+      await tx.run({ params: [], text: SCHEMA_LOCK_SQL }, { kind: 'system' });
+      for (const text of statements) {
+        await tx.run({ params: [], text }, { kind: 'system' });
+      }
+    });
+  } finally {
+    await executor.close();
+  }
+}

package/src/core/modules/migrate/sql-state-store.ts

@@ -0,0 +1,198 @@
+import type { DatabaseSqlEngine } from '../../common/interfaces/server-options.interface.js';
+import { createSqlExecutor, installedSqlEngine } from '../../common/persistence/sql/sql-executor.factory.js';
+import type { SqlExecutor, SqlTenantScope } from '../../common/persistence/sql/sql-executor.interface.js';
+import { quoteIdentifier, quoteMysqlIdentifier } from '../../common/persistence/sql/sql-naming.js';
+import type { MigrationSet, MigrationStateStore } from './migration-state-store.interface.js';
+
+const SYSTEM: SqlTenantScope = { kind: 'system' };
+
+/**
+ * Serialises migration runs of every replica: a transaction-scoped advisory lock, held by the transaction that
+ * surrounds the whole run. It ends with that transaction — and with its connection, so a replica killed mid-migration
+ * leaves nothing behind that another one would have to break (MongoDB needs a heartbeat and a stale-lock rule for
+ * that). A key of its own: the boot's schema evolution (`SCHEMA_LOCK_SQL`) must stay possible while a run waits.
+ */
+const MIGRATION_RUN_LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtext('lenne.tech:migrations'))";
+
+/**
+ * MySQL / MariaDB: a named lock per database, held by the session of the locking connection. It ends with that
+ * connection too, so a killed replica leaves nothing behind; released explicitly at the end of a run. The wait is
+ * bounded at a day — a run that takes longer is not waited for
+ */
+const MYSQL_RUN_LOCK_NAME = "CONCAT('lenne.tech:migrations:', SHA1(DATABASE()))";
+const MYSQL_RUN_LOCK_TIMEOUT_SECONDS = 86_400;
+
+export interface SqlStateStoreOptions {
+  /**
+   * SQL engine of the connection, the project's configured one. Without it the first INSTALLED engine is used
+   * (`typeorm`, `kysely`, `drizzle`): `createMigrationStore()` runs in the migrate CLI, which has no configuration to
+   * read it from. @default the first installed engine
+   */
+  engine?: DatabaseSqlEngine;
+  /** `false` runs migrations without the run lock — one replica only. @default true */
+  lock?: boolean;
+  /** Table holding the migration state. @default 'lt_migrations' */
+  tableName?: string;
+  /**
+   * Connection URL of the table OWNER (`database.ownerUrl`): migrations change the schema and the data of every
+   * tenant, which the application role may not
+   */
+  url: string;
+}
+
+/**
+ * Migration state on PostgreSQL, MySQL and MariaDB (the URL's scheme decides) — the counterpart of `MongoStateStore`:
+ * one row with the last run and the applied migrations, read and written by `MigrationRunner` and the `migrate` CLI,
+ * with every run serialised across replicas.
+ *
+ * Each call opens and closes its own connection, as `MongoStateStore` does: the CLI and the container entrypoint run it
+ * once per boot, and a pool kept open would hold the process.
+ */
+export class SqlStateStore implements MigrationStateStore {
+  readonly url: string;
+  protected readonly engine: DatabaseSqlEngine | undefined;
+  protected readonly locked: boolean;
+  /** MySQL and MariaDB share one dialect here */
+  protected readonly mysql: boolean;
+  protected readonly table: string;
+
+  constructor(options: SqlStateStoreOptions | string) {
+    const resolved = typeof options === 'string' ? { url: options } : options;
+    if (!resolved.url) {
+      throw new Error('SqlStateStore needs the connection URL of the table owner (database.ownerUrl)');
+    }
+    this.url = resolved.url;
+    this.engine = resolved.engine;
+    this.locked = resolved.lock !== false;
+    this.mysql = /^(mysql|mariadb):/i.test(resolved.url);
+    this.table = (this.mysql ? quoteMysqlIdentifier : quoteIdentifier)(resolved.tableName ?? 'lt_migrations');
+  }
+
+  load(fn: (err?: Error, set?: MigrationSet) => void): void {
+    this.loadAsync()
+      .then((result) => fn(undefined, result))
+      .catch((err) => fn(err));
+  }
+
+  async loadAsync(): Promise<MigrationSet> {
+    return this.request(async (executor) => {
+      const { rows } = await executor.run<{ last_run: null | string; migrations: MigrationSet['migrations'] | string }>(
+        {
+          params: [],
+          text: this.mysql
+            ? `SELECT \`last_run\`, \`migrations\` FROM ${this.table} WHERE \`id\` = 'state'`
+            : `SELECT "last_run", "migrations" FROM ${this.table} WHERE "id" = 'state'`,
+        },
+        SYSTEM,
+      );
+      if (!rows.length) {
+        console.debug('No migrations found, probably running the very first time');
+        // An empty object, as MongoStateStore returns for a first run
+        return {} as MigrationSet;
+      }
+      const migrations = rows[0].migrations;
+      return {
+        lastRun: rows[0].last_run ?? undefined,
+        // MariaDB's JSON is text where the driver does not recognise it
+        migrations: typeof migrations === 'string' ? JSON.parse(migrations) : migrations,
+      } as MigrationSet;
+    });
+  }
+
+  save(set: MigrationSet, fn: (err?: Error) => void): void {
+    this.saveAsync(set)
+      .then(() => fn())
+      .catch((err) => fn(err));
+  }
+
+  async saveAsync(set: MigrationSet): Promise<void> {
+    const { lastRun, migrations } = set;
+    await this.request((executor) =>
+      executor.run(
+        {
+          params: [lastRun ?? null, JSON.stringify(migrations ?? [])],
+          text: this.mysql
+            ? `INSERT INTO ${this.table} (\`id\`, \`last_run\`, \`migrations\`, \`updated_at\`) VALUES ('state', $1, $2, UTC_TIMESTAMP(3))
+              ON DUPLICATE KEY UPDATE \`last_run\` = $1, \`migrations\` = $2, \`updated_at\` = UTC_TIMESTAMP(3)`
+            : `INSERT INTO ${this.table} ("id", "last_run", "migrations", "updated_at") VALUES ('state', $1, $2::jsonb, now())
+              ON CONFLICT ("id") DO UPDATE SET "last_run" = EXCLUDED."last_run", "migrations" = EXCLUDED."migrations", "updated_at" = now()`,
+        },
+        SYSTEM,
+      ),
+    );
+  }
+
+  async withLock<T>(fn: () => Promise<T>): Promise<T> {
+    if (!this.locked) {
+      return fn();
+    }
+    const executor = await this.open();
+    try {
+      // The work itself runs on other connections (the runner reads and saves the state, migrations open their own);
+      // this transaction only holds the lock for as long as the work takes
+      return await executor.transaction(async (tx) => {
+        if (!this.mysql) {
+          await tx.run({ params: [], text: MIGRATION_RUN_LOCK_SQL }, SYSTEM);
+          return fn();
+        }
+        const { rows } = await tx.run<{ locked: number | string }>(
+          {
+            params: [MYSQL_RUN_LOCK_TIMEOUT_SECONDS],
+            text: `SELECT GET_LOCK(${MYSQL_RUN_LOCK_NAME}, $1) AS locked`,
+          },
+          SYSTEM,
+        );
+        if (Number(rows[0]?.locked) !== 1) {
+          throw new Error('Timed out waiting for the migration run of another process');
+        }
+        try {
+          return await fn();
+        } finally {
+          await tx
+            .run({ params: [], text: `SELECT RELEASE_LOCK(${MYSQL_RUN_LOCK_NAME})` }, SYSTEM)
+            .catch(() => undefined);
+        }
+      });
+    } finally {
+      await executor.close();
+    }
+  }
+
+  protected async request<R>(work: (executor: SqlExecutor) => Promise<R>): Promise<R> {
+    const executor = await this.open();
+    try {
+      return await work(executor);
+    } finally {
+      await executor.close();
+    }
+  }
+
+  /** A connection of its own, with the state table in place */
+  protected async open(): Promise<SqlExecutor> {
+    const executor = await createSqlExecutor({
+      applicationName: 'nest-server-migrations',
+      engine: this.engine ?? (await installedSqlEngine()),
+      maxConnections: 1,
+      url: this.url,
+    });
+    try {
+      await executor.run(
+        {
+          params: [],
+          text: this.mysql
+            ? `CREATE TABLE IF NOT EXISTS ${this.table} (\`id\` varchar(255) NOT NULL PRIMARY KEY, \`last_run\` longtext, \`migrations\` json NOT NULL, \`updated_at\` datetime(3) NOT NULL) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin`
+            : `CREATE TABLE IF NOT EXISTS ${this.table} ("id" text PRIMARY KEY, "last_run" text, "migrations" jsonb NOT NULL DEFAULT '[]'::jsonb, "updated_at" timestamptz NOT NULL DEFAULT now())`,
+        },
+        SYSTEM,
+      );
+    } catch (error) {
+      // Two replicas creating the table at the same moment: IF NOT EXISTS is not atomic in PostgreSQL, the loser fails
+      // with a duplicate of the table (42P07) or of its row type (23505) — and the table exists, which is all it asked
+      if (!['23505', '42P07'].includes((error as { code?: string })?.code)) {
+        await executor.close();
+        throw error;
+      }
+    }
+    return executor;
+  }
+}

package/src/core/modules/system-setup/README.md

@@ -172,6 +172,9 @@ NEST_SERVER_CONFIG='{ "systemSetup": { "initialAdmin": { "email": "admin@example
   After a SUCCESSFUL setup the marker REMAINS — setup must never run twice in a deployment. Test
   suites that reset the database to replay the setup flow therefore have to drop `system-setup-locks`
   alongside `users`; clearing only `users` leaves setup refused until the claim goes stale (5 min).
+- **On a PostgreSQL default source** (`database`) the claim is the row `system-setup:initial-admin` in
+  the lease table `lt_locks`, with the same semantics: one replica wins, released on failure, kept
+  after success, taken over once stale. Where this README names `system-setup-locks`, read that row.
 
 ### Security Best Practices
 
@@ -248,8 +251,10 @@ confirms a state that is not the problem, and step 2 confirms an empty database
 empty.
 
 1. Look at the server log. Since 11.36.5 the service logs a warning naming the collection, the
-   release-on-failure semantics and the stale window.
-2. Check the collection directly: `db.getCollection('system-setup-locks').find()`
+   release-on-failure semantics and the stale window — on a PostgreSQL default source the
+   `lt_locks` row instead.
+2. Check the collection directly: `db.getCollection('system-setup-locks').find()` — on PostgreSQL
+   `SELECT * FROM lt_locks WHERE id = 'system-setup:initial-admin'`
 3. If you reset the database to replay the setup, drop `system-setup-locks` alongside `users` -
    see **Resetting for tests** below. Otherwise wait for the claim to go stale (5 minutes after it
    was taken).
@@ -267,6 +272,13 @@ for (const col of ['users', 'account', 'session', 'system-setup-locks']) {
 }
 ```
 
+On a PostgreSQL default source, the same reset (run there by the same suite in the `postgres` project):
+
+```sql
+DELETE FROM session; DELETE FROM account; DELETE FROM users;
+DELETE FROM lt_locks WHERE id = 'system-setup:initial-admin';
+```
+
 ### Init returns 403 "System setup requires BetterAuth"
 
 **Cause:** BetterAuth is not configured or not enabled.