@lenne.tech/nest-server 11.32.4 → 11.33.0

package/src/core/modules/hub/services/hub-log-buffer.service.ts

@@ -7,12 +7,14 @@ import {
   LogLevel,
   OnModuleDestroy,
   OnModuleInit,
+  Optional,
 } from '@nestjs/common';
 import { inspect } from 'util';
 
 import { redactSensitiveText } from '../../../common/helpers/logging.helper';
+import { CoreRedisService } from '../../../common/services/core-redis.service';
 import { HUB_CONFIG } from '../hub.constants';
-import { HubRingBuffer } from '../hub-ring-buffer';
+import { HubBuffer } from '../hub-buffer';
 import { HubLogRecord, HubLogsData } from '../interfaces/hub-panels.interface';
 import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
 
@@ -90,22 +92,32 @@ class HubDelegatingLogger implements LoggerService {
  */
 @Injectable()
 export class HubLogBufferService implements OnModuleDestroy, OnModuleInit {
-  private buffer?: HubRingBuffer<BufferedLog>;
+  private buffer?: HubBuffer<BufferedLog>;
   private cfg?: Exclude<ResolvedHubConfig['collectors']['logs'], false>;
   private delegating?: HubDelegatingLogger;
   private excludeContexts = new Set<string>();
+
+  /**
+   * Set by onModuleDestroy. The Nest Logger override is PROCESS-GLOBAL, and with a shutdown
+   * delay the HTTP server keeps serving while the app drains — so a Hub poll arriving in that
+   * window could re-install a logger belonging to an app that no longer exists.
+   */
+  private destroyed = false;
   private previousRef?: LoggerService;
 
-  constructor(@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig) {}
+  constructor(
+    @Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
+    @Optional() protected readonly redis?: CoreRedisService,
+  ) {}
 
   /** Idempotent install of the delegating logger. Public so reads can self-heal after a foreign override. */
   attach(): void {
-    if (this.config.collectors.logs === false) {
+    if (this.destroyed || this.config.collectors.logs === false) {
       return;
     }
     if (!this.cfg) {
       this.cfg = this.config.collectors.logs;
-      this.buffer = new HubRingBuffer<BufferedLog>(this.cfg.capacity);
+      this.buffer = new HubBuffer<BufferedLog>(this.cfg.capacity, 'logs', this.redis);
       this.excludeContexts = new Set(this.cfg.excludeContexts);
     }
     const current = this.staticRef();
@@ -149,13 +161,17 @@ export class HubLogBufferService implements OnModuleDestroy, OnModuleInit {
     return this.config.collectors.logs !== false;
   }
 
-  getData(since?: number): HubLogsData {
+  async getData(since?: number): Promise<HubLogsData> {
     this.selfHeal();
-    const records = !this.buffer ? [] : since === undefined ? this.buffer.recent() : this.buffer.since(since);
-    return { cursor: this.buffer?.lastSeq ?? -1, dropped: this.buffer?.firstRetainedSeq ?? -1, records };
+    if (!this.buffer) {
+      return { cursor: -1, dropped: -1, records: [] };
+    }
+    const { cursor, dropped, entries } = await this.buffer.read(since);
+    return { cursor, dropped, records: entries };
   }
 
   onModuleDestroy(): void {
+    this.destroyed = true;
     this.detach();
   }
 

package/src/core/modules/hub/services/hub-query-profiler.service.ts

@@ -3,8 +3,9 @@ import { InjectConnection } from '@nestjs/mongoose';
 import { Connection } from 'mongoose';
 
 import { redactSensitiveText } from '../../../common/helpers/logging.helper';
+import { CoreRedisService } from '../../../common/services/core-redis.service';
 import { HUB_CONFIG, HUB_QUERY_PENDING_LIMIT } from '../hub.constants';
-import { HubRingBuffer } from '../hub-ring-buffer';
+import { HubBuffer } from '../hub-buffer';
 import { normalizeCommandShape } from '../helpers/hub-command-shape.helper';
 import { HubQueriesData, HubQueryRecord, HubQueryTemplate } from '../interfaces/hub-panels.interface';
 import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
@@ -58,7 +59,7 @@ const SHAPED_FIELDS = ['filter', 'query', 'pipeline', 'sort', 'projection', 'q',
 export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
   protected readonly logger = new Logger(HubQueryProfilerService.name);
 
-  private buffer?: HubRingBuffer<ProfilerRecord>;
+  private buffer?: HubBuffer<ProfilerRecord>;
   private client?: {
     off?: (e: string, l: (...a: any[]) => void) => void;
     on: (e: string, l: (...a: any[]) => void) => void;
@@ -78,6 +79,7 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
   constructor(
     @Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
     @Optional() @InjectConnection() protected readonly connection?: Connection,
+    @Optional() protected readonly redis?: CoreRedisService,
   ) {}
 
   /** Clear the buffer (Hub action). */
@@ -90,10 +92,11 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
     return this.config.collectors.queries !== false;
   }
 
-  getData(): HubQueriesData {
-    const records = this.buffer?.recent() ?? [];
+  async getData(): Promise<HubQueriesData> {
+    const data = this.buffer ? await this.buffer.read() : { cursor: -1, entries: [] };
+    const records = data.entries;
     return {
-      cursor: this.buffer?.lastSeq ?? -1,
+      cursor: data.cursor,
       recent: records.slice(-100),
       slowest: [...records].sort((a, b) => b.durationMs - a.durationMs).slice(0, 10),
       summary: this.summary(records),
@@ -118,7 +121,7 @@ export class HubQueryProfilerService implements OnModuleDestroy, OnModuleInit {
       return; // zero cost when disabled
     }
     this.cfg = this.config.collectors.queries;
-    this.buffer = new HubRingBuffer<ProfilerRecord>(this.cfg.capacity);
+    this.buffer = new HubBuffer<ProfilerRecord>(this.cfg.capacity, 'queries', this.redis);
     if (this.cfg.ignoreCommands?.length) {
       this.ignore = new Set(this.cfg.ignoreCommands.map((c) => c.toLowerCase()));
     }

package/src/core/modules/hub/services/hub-trace-buffer.service.ts

@@ -1,8 +1,9 @@
-import { Inject, Injectable } from '@nestjs/common';
+import { Inject, Injectable, Optional } from '@nestjs/common';
 import { Request, Response } from 'express';
 
+import { CoreRedisService } from '../../../common/services/core-redis.service';
 import { HUB_CONFIG } from '../hub.constants';
-import { HubRingBuffer } from '../hub-ring-buffer';
+import { HubBuffer } from '../hub-buffer';
 import { HubTraceRecord, HubTracesData } from '../interfaces/hub-panels.interface';
 import { ResolvedHubConfig } from '../interfaces/hub-config.interface';
 
@@ -19,14 +20,17 @@ interface TraceRecord extends HubTraceRecord {
  */
 @Injectable()
 export class HubTraceBufferService {
-  private readonly buffer?: HubRingBuffer<TraceRecord>;
+  private readonly buffer?: HubBuffer<TraceRecord>;
   private readonly cfg?: Exclude<ResolvedHubConfig['collectors']['traces'], false>;
   private readonly excludePrefixes: string[];
 
-  constructor(@Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig) {
+  constructor(
+    @Inject(HUB_CONFIG) protected readonly config: ResolvedHubConfig,
+    @Optional() protected readonly redis?: CoreRedisService,
+  ) {
     if (config.collectors.traces !== false) {
       this.cfg = config.collectors.traces;
-      this.buffer = new HubRingBuffer<TraceRecord>(this.cfg.capacity);
+      this.buffer = new HubBuffer<TraceRecord>(this.cfg.capacity, 'traces', redis);
     }
     // Always exclude the Hub's own routes so polling does not flood the trace list with self-noise.
     this.excludePrefixes = [...(this.cfg?.excludePaths ?? []), '/' + config.path];
@@ -40,8 +44,9 @@ export class HubTraceBufferService {
     return this.config.collectors.traces !== false;
   }
 
-  getData(since?: number): HubTracesData {
-    const records = !this.buffer ? [] : since === undefined ? this.buffer.recent() : this.buffer.since(since);
+  async getData(since?: number): Promise<HubTracesData> {
+    const data = this.buffer ? await this.buffer.read(since) : { cursor: -1, dropped: -1, entries: [] };
+    const records = data.entries;
     let totalMs = 0;
     let slowCount = 0;
     let errorCount = 0;
@@ -55,8 +60,8 @@ export class HubTraceBufferService {
       }
     }
     return {
-      cursor: this.buffer?.lastSeq ?? -1,
-      dropped: this.buffer?.firstRetainedSeq ?? -1,
+      cursor: data.cursor,
+      dropped: data.dropped,
       summary: { avgMs: records.length ? totalMs / records.length : 0, errorCount, slowCount, total: records.length },
       traces: records,
     };

package/src/core/modules/migrate/README.md

@@ -230,7 +230,7 @@ The migration state is stored in a MongoDB collection (default: `migrations`) as
 
 ### Locking Mechanism
 
-When using `synchronizedMigration` or `synchronizedUp`:
+How the lock works, regardless of which entry point uses it:
 
 1. A unique index is created on the lock collection
 2. The migration process attempts to insert a lock document
@@ -240,6 +240,27 @@ When using `synchronizedMigration` or `synchronizedUp`:
 
 This ensures that in a cluster with multiple nodes, migrations run on only one machine at a time.
 
+**Active by default for `migrate up`.** Stores built by `createMigrationStore()` use the
+lock collection `migrations_lock` unless another name is given, and `MigrationRunner.up()`
+(the CLI's `up` command) acquires that lock around the whole run. This matters because the
+container entrypoint runs migrations on **every** boot: without the lock, N replicas
+starting together each read the same empty state and apply the same pending migration N
+times. A replica that waited re-reads the state inside the lock and finds nothing pending.
+
+With a single replica nothing changes — the lock is acquired and released uncontended.
+
+Opt out by passing an empty lock collection name:
+
+```javascript
+module.exports = createMigrationStore(uri, 'migrations', ''); // no locking
+```
+
+A `MongoStateStore` constructed **directly** still locks only when you pass
+`lockCollectionName` yourself; the default lives in `createMigrationStore()`.
+
+The legacy `synchronizedMigration` / `synchronizedUp` helpers use the same lock and still
+require `lockCollectionName` to be set explicitly (they throw otherwise).
+
 ## Examples
 
 ### Example: Migration File
@@ -356,11 +377,17 @@ Factory function to create a migration store class for use with the migrate CLI:
 const { createMigrationStore } = require('@lenne.tech/nest-server');
 const config = require('../src/config.env');
 
-module.exports = createMigrationStore(
-  config.default.mongoose.uri,
-  'migrations', // optional collection name
-  'migration_lock', // optional lock collection for clusters
-);
+module.exports = createMigrationStore(config.default.mongoose.uri);
+```
+
+**Locking is on by default.** The third parameter defaults to `'migrations_lock'`, so
+`migrate up` is serialized across replicas without any project change (see
+[Locking Mechanism](#locking-mechanism)):
+
+```javascript
+createMigrationStore(uri); // collection 'migrations', lock 'migrations_lock'
+createMigrationStore(uri, 'migrations', 'custom_lock'); // custom lock collection
+createMigrationStore(uri, 'migrations', ''); // opt out: no locking
 ```
 
 ### getDb()

package/src/core/modules/migrate/helpers/migration.helper.ts

@@ -2,6 +2,8 @@ import * as fs from 'fs';
 import { Db, GridFSBucket, MongoClient, ObjectId } from 'mongodb';
 import * as path from 'path';
 
+import { MongoStateStore } from '../mongo-state-store';
+
 /**
  * Migration helper functions for database operations
  */
@@ -247,9 +249,13 @@ export const uploadFileToGridFS = async (
 /**
  * Create a migration state store factory
  *
+ * The lock collection is set by DEFAULT, so `migrate up` from N replicas booting at
+ * the same time serializes instead of applying the same migration twice (the container
+ * entrypoint runs migrations on every boot). Pass an empty string to opt out.
+ *
  * @param mongoUrl - MongoDB connection URI
  * @param collectionName - Optional collection name (default: 'migrations')
- * @param lockCollectionName - Optional lock collection name for cluster environments
+ * @param lockCollectionName - Lock collection name (default: 'migrations_lock'); `''` disables locking
  * @returns MongoStateStore class that can be used with migrate CLI
  *
  * @example
@@ -264,15 +270,16 @@ export const uploadFileToGridFS = async (
 export const createMigrationStore = (
   mongoUrl: string,
   collectionName: string = 'migrations',
-  lockCollectionName?: string,
-) => {
-  const { MongoStateStore } = require('../mongo-state-store');
-
+  lockCollectionName: string = 'migrations_lock',
+  // Explicit return type: the returned class is anonymous to the declaration emitter, and
+  // MongoStateStore has private members, so an inferred type cannot be written to the .d.ts
+  // (TS4094).
+): new () => MongoStateStore => {
   return class MigrationStateStore extends MongoStateStore {
     constructor() {
       super({
         collectionName,
-        lockCollectionName,
+        lockCollectionName: lockCollectionName || undefined,
         uri: mongoUrl,
       });
     }

package/src/core/modules/migrate/migration-runner.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { MongoStateStore } from './mongo-state-store';
+import { MongoStateStore, withMigrationLock } from './mongo-state-store';
 
 /**
  * Migration file interface
@@ -221,8 +221,17 @@ export class MigrationRunner {
 
   /**
    * Run all pending migrations (up)
+   *
+   * Serialized via the state store's lock collection when it has one (the default for
+   * stores built by `createMigrationStore()`). Reading the state INSIDE the lock is what
+   * makes it safe: a replica that waited sees the migrations the holder just applied and
+   * finds nothing pending, instead of applying them a second time.
    */
   async up(): Promise<void> {
+    await withMigrationLock(this.options.stateStore, () => this.runUp());
+  }
+
+  protected async runUp(): Promise<void> {
     const { _endMigration, _startMigration } = await import('./helpers/migration.helper');
 
     const allMigrations = await this.loadMigrationFiles();

package/src/core/modules/migrate/mongo-state-store.ts

@@ -1,4 +1,6 @@
+import { randomUUID } from 'crypto';
 import { Db, MongoClient } from 'mongodb';
+import { hostname } from 'os';
 import { promisify } from 'util';
 
 /**
@@ -181,14 +183,20 @@ export async function synchronizedMigration(
     throw new Error('`lockCollectionName` in MongoStateStore is not set');
   }
 
+  let heartbeat: undefined | { stop: () => void };
+  let lockToken: string | undefined;
   try {
-    await acquireLock(stateStore.mongodbHost, lockCollectionName);
+    lockToken = await acquireLock(stateStore.mongodbHost, lockCollectionName);
+    heartbeat = startLockHeartbeat(stateStore.mongodbHost, lockCollectionName, lockToken);
 
     // Load migration set using async method
     const set = await stateStore.loadAsync();
     await callback(set);
   } finally {
-    await releaseLock(stateStore.mongodbHost, lockCollectionName);
+    heartbeat?.stop();
+    if (lockToken) {
+      await releaseLock(stateStore.mongodbHost, lockCollectionName, lockToken);
+    }
   }
 }
 
@@ -217,14 +225,119 @@ export async function synchronizedUp(opts: MigrationOptions): Promise<void> {
   });
 }
 
+/**
+ * Runs `fn` while holding the migration lock of the given state store
+ *
+ * When the store has no `lockCollectionName`, `fn` runs unsynchronized — a single
+ * replica keeps behaving exactly as before. Stores built by `createMigrationStore()`
+ * carry the default lock collection, so concurrent `migrate up` runs from several
+ * replicas serialize here: the second replica waits, then re-reads the migration
+ * state and finds nothing pending.
+ *
+ * @param stateStore - State store holding the connection URI and lock collection name
+ * @param fn - Work to execute under the lock
+ * @returns Promise with the result of `fn`
+ */
+export async function withMigrationLock<T>(stateStore: MongoStateStore, fn: () => Promise<T>): Promise<T> {
+  const lockCollectionName = stateStore.lockCollectionName;
+
+  if (!lockCollectionName) {
+    return fn();
+  }
+
+  const token = await acquireLock(stateStore.mongodbHost, lockCollectionName);
+  const heartbeat = startLockHeartbeat(stateStore.mongodbHost, lockCollectionName, token);
+  try {
+    return await fn();
+  } finally {
+    heartbeat.stop();
+    await releaseLock(stateStore.mongodbHost, lockCollectionName, token);
+  }
+}
+
+/**
+ * How long a lock may go without a heartbeat before another replica may break it.
+ *
+ * Only a lock whose holder stopped refreshing it is ever broken, so this is not a cap
+ * on migration runtime — a migration running for hours keeps its lock as long as the
+ * process lives.
+ */
+const LOCK_STALE_AFTER_MS = 60_000;
+
+/** Heartbeat interval — comfortably below {@link LOCK_STALE_AFTER_MS} */
+const LOCK_HEARTBEAT_INTERVAL_MS = 15_000;
+
+/** How long a replica waits for a held lock before giving up with a diagnosable error */
+const LOCK_WAIT_TIMEOUT_MS = 15 * 60_000;
+
+/**
+ * Keep refreshing `acquiredAt` while the migration runs.
+ *
+ * This is what makes lock-breaking safe: a lock is only ever taken over when its holder
+ * stopped refreshing it, which for a live process cannot happen. Without the heartbeat,
+ * breaking a stale lock and a long-running migration would be indistinguishable.
+ */
+function startLockHeartbeat(url: string, lockCollectionName: string, token: string): { stop: () => void } {
+  // ONE connection for the whole heartbeat, rather than `dbRequest()`'s connect-and-close per
+  // call: that opened and tore down a MongoClient every 15 seconds for the entire migration run —
+  // a full handshake each time, against the database at its busiest moment. The client is created
+  // on the first tick (so a heartbeat that never fires costs nothing) and closed by stop().
+  let client: MongoClient | undefined;
+  let stopped = false;
+
+  const beat = async (): Promise<void> => {
+    if (stopped) {
+      return;
+    }
+    if (!client) {
+      const connected = await MongoClient.connect(url);
+      if (stopped) {
+        // stop() ran while we were connecting — nothing would ever close this one.
+        await connected.close();
+        return;
+      }
+      client = connected;
+    }
+    // Scoped: once our lock has been broken and re-taken, our heartbeat must not keep the NEW
+    // holder's lock alive on our behalf.
+    await client
+      .db()
+      .collection(lockCollectionName)
+      .updateOne({ lock: 'lock', owner: token }, { $set: { acquiredAt: new Date() } });
+  };
+
+  const timer = setInterval(() => {
+    beat().catch((error) => {
+      // A missed heartbeat is not fatal on its own — the next one may succeed, and only
+      // a sustained gap makes the lock breakable. Drop the connection though: it may itself be
+      // what failed, and a held-open broken client would fail every remaining tick.
+      console.warn(`Migration lock heartbeat failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      const broken = client;
+      client = undefined;
+      void broken?.close().catch(() => undefined);
+    });
+  }, LOCK_HEARTBEAT_INTERVAL_MS);
+  timer.unref?.();
+
+  return {
+    stop: () => {
+      stopped = true;
+      clearInterval(timer);
+      const open = client;
+      client = undefined;
+      void open?.close().catch(() => undefined);
+    },
+  };
+}
+
 /**
  * Acquires a lock in MongoDB to ensure only one migration runs at a time
  *
  * @param url - MongoDB connection URI
  * @param lockCollectionName - Name of the collection to use for locking
  */
-async function acquireLock(url: string, lockCollectionName: string): Promise<void> {
-  await dbRequest(url, async (db) => {
+async function acquireLock(url: string, lockCollectionName: string): Promise<string> {
+  return dbRequest(url, async (db) => {
     const collection = db.collection(lockCollectionName);
 
     // Create unique index for atomicity
@@ -233,15 +346,58 @@ async function acquireLock(url: string, lockCollectionName: string): Promise<voi
     // https://stackoverflow.com/questions/33346175/mongodb-upsert-operation-seems-not-atomic-which-throws-duplicatekeyexception/34784533
     await collection.createIndex({ lock: 1 }, { unique: true });
 
+    const token = lockOwnerId();
+    const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS;
     let showMessage = true;
 
     for (;;) {
-      // Use updateOne with upsert for atomic lock acquisition (same as original package)
-      const result = await collection.updateOne({ lock: 'lock' }, { $set: { lock: 'lock' } }, { upsert: true });
-      const lockAcquired = result.upsertedCount > 0;
+      // Use updateOne with upsert for atomic lock acquisition (same as original package).
+      // `$setOnInsert` stamps the acquisition time only for the winner, so a loser's
+      // update never refreshes the holder's staleness clock.
+      const result = await collection.updateOne(
+        { lock: 'lock' },
+        { $set: { lock: 'lock' }, $setOnInsert: { acquiredAt: new Date(), owner: token } },
+        { upsert: true },
+      );
+
+      if (result.upsertedCount > 0) {
+        return token;
+      }
+
+      // The holder may be gone: migrations run on every container boot, so a replica
+      // SIGKILLed mid-migration (OOM, node drain, failed deploy) would otherwise leave a
+      // lock nobody holds — and every future boot of every replica would wait on it
+      // forever. Only a lock that stopped heart-beating is broken; see LOCK_STALE_AFTER_MS.
+      const holder = await collection.findOne({ lock: 'lock' });
+      if (holder) {
+        const acquiredAt = holder.acquiredAt instanceof Date ? holder.acquiredAt : undefined;
+        if (!acquiredAt) {
+          // Written by a version that did not stamp the lock. Start its clock now rather
+          // than breaking it immediately — the holder may well be alive.
+          await collection.updateOne(
+            { _id: holder._id, acquiredAt: { $exists: false } },
+            { $set: { acquiredAt: new Date() } },
+          );
+        } else if (Date.now() - acquiredAt.getTime() > LOCK_STALE_AFTER_MS) {
+          console.warn(
+            `Breaking stale migration lock in "${lockCollectionName}" (last heartbeat ${acquiredAt.toISOString()}, ` +
+              `owner ${holder.owner ?? 'unknown'}) — its holder is gone.`,
+          );
+          // Matching on acquiredAt makes the break safe under concurrency: if another
+          // waiter already broke and re-acquired the lock, the timestamp differs and this
+          // deletes nothing.
+          await collection.deleteOne({ _id: holder._id, acquiredAt });
+          continue;
+        }
+      }
 
-      if (lockAcquired) {
-        break;
+      if (Date.now() > deadline) {
+        throw new Error(
+          `Timed out after ${Math.round(LOCK_WAIT_TIMEOUT_MS / 60_000)} minutes waiting for the migration lock in ` +
+            `collection "${lockCollectionName}". Another replica is still migrating, or the lock is held by a process ` +
+            `that is alive but stuck. Inspect it with: db.getCollection("${lockCollectionName}").find({}) — and remove ` +
+            `the document only once you are sure no migration is running.`,
+        );
       }
 
       if (showMessage) {
@@ -254,6 +410,15 @@ async function acquireLock(url: string, lockCollectionName: string): Promise<voi
   });
 }
 
+/**
+ * Identifies the process holding the lock, so a stale-lock warning names something actionable
+ */
+function lockOwnerId(): string {
+  // Random suffix, not just host+pid: a restarted process can reuse a pid, and the token is what
+  // decides whose lock a release may delete.
+  return `${hostname()}:${process.pid}:${randomUUID()}`;
+}
+
 /**
  * Executes database operations with automatic connection management
  *
@@ -278,6 +443,9 @@ async function dbRequest<T>(url: string, callback: (db: Db) => Promise<T> | T):
  * @param url - MongoDB connection URI
  * @param lockCollectionName - Name of the collection used for locking
  */
-async function releaseLock(url: string, lockCollectionName: string): Promise<void> {
-  await dbRequest(url, (db) => db.collection(lockCollectionName).deleteOne({ lock: 'lock' }));
+async function releaseLock(url: string, lockCollectionName: string, token: string): Promise<void> {
+  // Scoped to OUR token. An unscoped delete removes whatever lock exists — so after a missed
+  // heartbeat let another replica break and re-take the lock, the original holder's `finally`
+  // would delete the NEW holder's lock and let a third replica start a concurrent migration.
+  await dbRequest(url, (db) => db.collection(lockCollectionName).deleteOne({ lock: 'lock', owner: token }));
 }

package/src/core/modules/system-setup/README.md

@@ -164,7 +164,11 @@ NEST_SERVER_CONFIG='{ "systemSetup": { "initialAdmin": { "email": "admin@example
 - The admin is created automatically during application bootstrap (`OnApplicationBootstrap`)
 - Same zero-user guard applies: only works when no users exist
 - If users already exist, auto-creation is silently skipped (no error)
-- Race conditions between multiple instances are handled gracefully
+- Race conditions between multiple instances are handled gracefully: the zero-user check alone is
+  check-then-act, so N replicas booting at the same time would each pass it. Before creating, an
+  instance claims the setup by upserting the marker `{ _id: 'initial-admin' }` into the
+  `system-setup-locks` collection — an atomic operation only one instance wins. The others log a
+  debug message and skip. If creation fails, the marker is removed again so a later boot can retry.
 
 ### Security Best Practices
 
@@ -179,7 +183,8 @@ NEST_SERVER_CONFIG='{ "systemSetup": { "initialAdmin": { "email": "admin@example
 
 1. **Zero-user guard** - Init only works when `countDocuments({}) === 0`
 2. **Enabled by default** - Safe because endpoints are permanently locked once any user exists
-3. **Race condition protection** - MongoDB unique email index prevents duplicates
+3. **Race condition protection** - MongoDB unique email index prevents duplicates; auto-creation on
+   bootstrap is additionally claimed atomically via the `system-setup-locks` marker
 4. **Permanent lock** - Once any user exists, init returns 403
 5. **BetterAuth required** - Returns 403 if BetterAuth is not enabled