@lenne.tech/nest-server 11.32.3 → 11.33.0

package/src/core/modules/migrate/mongo-state-store.ts

@@ -1,4 +1,6 @@
+import { randomUUID } from 'crypto';
 import { Db, MongoClient } from 'mongodb';
+import { hostname } from 'os';
 import { promisify } from 'util';
 
 /**
@@ -181,14 +183,20 @@ export async function synchronizedMigration(
     throw new Error('`lockCollectionName` in MongoStateStore is not set');
   }
 
+  let heartbeat: undefined | { stop: () => void };
+  let lockToken: string | undefined;
   try {
-    await acquireLock(stateStore.mongodbHost, lockCollectionName);
+    lockToken = await acquireLock(stateStore.mongodbHost, lockCollectionName);
+    heartbeat = startLockHeartbeat(stateStore.mongodbHost, lockCollectionName, lockToken);
 
     // Load migration set using async method
     const set = await stateStore.loadAsync();
     await callback(set);
   } finally {
-    await releaseLock(stateStore.mongodbHost, lockCollectionName);
+    heartbeat?.stop();
+    if (lockToken) {
+      await releaseLock(stateStore.mongodbHost, lockCollectionName, lockToken);
+    }
   }
 }
 
@@ -217,14 +225,119 @@ export async function synchronizedUp(opts: MigrationOptions): Promise<void> {
   });
 }
 
+/**
+ * Runs `fn` while holding the migration lock of the given state store
+ *
+ * When the store has no `lockCollectionName`, `fn` runs unsynchronized — a single
+ * replica keeps behaving exactly as before. Stores built by `createMigrationStore()`
+ * carry the default lock collection, so concurrent `migrate up` runs from several
+ * replicas serialize here: the second replica waits, then re-reads the migration
+ * state and finds nothing pending.
+ *
+ * @param stateStore - State store holding the connection URI and lock collection name
+ * @param fn - Work to execute under the lock
+ * @returns Promise with the result of `fn`
+ */
+export async function withMigrationLock<T>(stateStore: MongoStateStore, fn: () => Promise<T>): Promise<T> {
+  const lockCollectionName = stateStore.lockCollectionName;
+
+  if (!lockCollectionName) {
+    return fn();
+  }
+
+  const token = await acquireLock(stateStore.mongodbHost, lockCollectionName);
+  const heartbeat = startLockHeartbeat(stateStore.mongodbHost, lockCollectionName, token);
+  try {
+    return await fn();
+  } finally {
+    heartbeat.stop();
+    await releaseLock(stateStore.mongodbHost, lockCollectionName, token);
+  }
+}
+
+/**
+ * How long a lock may go without a heartbeat before another replica may break it.
+ *
+ * Only a lock whose holder stopped refreshing it is ever broken, so this is not a cap
+ * on migration runtime — a migration running for hours keeps its lock as long as the
+ * process lives.
+ */
+const LOCK_STALE_AFTER_MS = 60_000;
+
+/** Heartbeat interval — comfortably below {@link LOCK_STALE_AFTER_MS} */
+const LOCK_HEARTBEAT_INTERVAL_MS = 15_000;
+
+/** How long a replica waits for a held lock before giving up with a diagnosable error */
+const LOCK_WAIT_TIMEOUT_MS = 15 * 60_000;
+
+/**
+ * Keep refreshing `acquiredAt` while the migration runs.
+ *
+ * This is what makes lock-breaking safe: a lock is only ever taken over when its holder
+ * stopped refreshing it, which for a live process cannot happen. Without the heartbeat,
+ * breaking a stale lock and a long-running migration would be indistinguishable.
+ */
+function startLockHeartbeat(url: string, lockCollectionName: string, token: string): { stop: () => void } {
+  // ONE connection for the whole heartbeat, rather than `dbRequest()`'s connect-and-close per
+  // call: that opened and tore down a MongoClient every 15 seconds for the entire migration run —
+  // a full handshake each time, against the database at its busiest moment. The client is created
+  // on the first tick (so a heartbeat that never fires costs nothing) and closed by stop().
+  let client: MongoClient | undefined;
+  let stopped = false;
+
+  const beat = async (): Promise<void> => {
+    if (stopped) {
+      return;
+    }
+    if (!client) {
+      const connected = await MongoClient.connect(url);
+      if (stopped) {
+        // stop() ran while we were connecting — nothing would ever close this one.
+        await connected.close();
+        return;
+      }
+      client = connected;
+    }
+    // Scoped: once our lock has been broken and re-taken, our heartbeat must not keep the NEW
+    // holder's lock alive on our behalf.
+    await client
+      .db()
+      .collection(lockCollectionName)
+      .updateOne({ lock: 'lock', owner: token }, { $set: { acquiredAt: new Date() } });
+  };
+
+  const timer = setInterval(() => {
+    beat().catch((error) => {
+      // A missed heartbeat is not fatal on its own — the next one may succeed, and only
+      // a sustained gap makes the lock breakable. Drop the connection though: it may itself be
+      // what failed, and a held-open broken client would fail every remaining tick.
+      console.warn(`Migration lock heartbeat failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      const broken = client;
+      client = undefined;
+      void broken?.close().catch(() => undefined);
+    });
+  }, LOCK_HEARTBEAT_INTERVAL_MS);
+  timer.unref?.();
+
+  return {
+    stop: () => {
+      stopped = true;
+      clearInterval(timer);
+      const open = client;
+      client = undefined;
+      void open?.close().catch(() => undefined);
+    },
+  };
+}
+
 /**
  * Acquires a lock in MongoDB to ensure only one migration runs at a time
  *
  * @param url - MongoDB connection URI
  * @param lockCollectionName - Name of the collection to use for locking
  */
-async function acquireLock(url: string, lockCollectionName: string): Promise<void> {
-  await dbRequest(url, async (db) => {
+async function acquireLock(url: string, lockCollectionName: string): Promise<string> {
+  return dbRequest(url, async (db) => {
     const collection = db.collection(lockCollectionName);
 
     // Create unique index for atomicity
@@ -233,15 +346,58 @@ async function acquireLock(url: string, lockCollectionName: string): Promise<voi
     // https://stackoverflow.com/questions/33346175/mongodb-upsert-operation-seems-not-atomic-which-throws-duplicatekeyexception/34784533
     await collection.createIndex({ lock: 1 }, { unique: true });
 
+    const token = lockOwnerId();
+    const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS;
     let showMessage = true;
 
     for (;;) {
-      // Use updateOne with upsert for atomic lock acquisition (same as original package)
-      const result = await collection.updateOne({ lock: 'lock' }, { $set: { lock: 'lock' } }, { upsert: true });
-      const lockAcquired = result.upsertedCount > 0;
+      // Use updateOne with upsert for atomic lock acquisition (same as original package).
+      // `$setOnInsert` stamps the acquisition time only for the winner, so a loser's
+      // update never refreshes the holder's staleness clock.
+      const result = await collection.updateOne(
+        { lock: 'lock' },
+        { $set: { lock: 'lock' }, $setOnInsert: { acquiredAt: new Date(), owner: token } },
+        { upsert: true },
+      );
+
+      if (result.upsertedCount > 0) {
+        return token;
+      }
+
+      // The holder may be gone: migrations run on every container boot, so a replica
+      // SIGKILLed mid-migration (OOM, node drain, failed deploy) would otherwise leave a
+      // lock nobody holds — and every future boot of every replica would wait on it
+      // forever. Only a lock that stopped heart-beating is broken; see LOCK_STALE_AFTER_MS.
+      const holder = await collection.findOne({ lock: 'lock' });
+      if (holder) {
+        const acquiredAt = holder.acquiredAt instanceof Date ? holder.acquiredAt : undefined;
+        if (!acquiredAt) {
+          // Written by a version that did not stamp the lock. Start its clock now rather
+          // than breaking it immediately — the holder may well be alive.
+          await collection.updateOne(
+            { _id: holder._id, acquiredAt: { $exists: false } },
+            { $set: { acquiredAt: new Date() } },
+          );
+        } else if (Date.now() - acquiredAt.getTime() > LOCK_STALE_AFTER_MS) {
+          console.warn(
+            `Breaking stale migration lock in "${lockCollectionName}" (last heartbeat ${acquiredAt.toISOString()}, ` +
+              `owner ${holder.owner ?? 'unknown'}) — its holder is gone.`,
+          );
+          // Matching on acquiredAt makes the break safe under concurrency: if another
+          // waiter already broke and re-acquired the lock, the timestamp differs and this
+          // deletes nothing.
+          await collection.deleteOne({ _id: holder._id, acquiredAt });
+          continue;
+        }
+      }
 
-      if (lockAcquired) {
-        break;
+      if (Date.now() > deadline) {
+        throw new Error(
+          `Timed out after ${Math.round(LOCK_WAIT_TIMEOUT_MS / 60_000)} minutes waiting for the migration lock in ` +
+            `collection "${lockCollectionName}". Another replica is still migrating, or the lock is held by a process ` +
+            `that is alive but stuck. Inspect it with: db.getCollection("${lockCollectionName}").find({}) — and remove ` +
+            `the document only once you are sure no migration is running.`,
+        );
       }
 
       if (showMessage) {
@@ -254,6 +410,15 @@ async function acquireLock(url: string, lockCollectionName: string): Promise<voi
   });
 }
 
+/**
+ * Identifies the process holding the lock, so a stale-lock warning names something actionable
+ */
+function lockOwnerId(): string {
+  // Random suffix, not just host+pid: a restarted process can reuse a pid, and the token is what
+  // decides whose lock a release may delete.
+  return `${hostname()}:${process.pid}:${randomUUID()}`;
+}
+
 /**
  * Executes database operations with automatic connection management
  *
@@ -278,6 +443,9 @@ async function dbRequest<T>(url: string, callback: (db: Db) => Promise<T> | T):
  * @param url - MongoDB connection URI
  * @param lockCollectionName - Name of the collection used for locking
  */
-async function releaseLock(url: string, lockCollectionName: string): Promise<void> {
-  await dbRequest(url, (db) => db.collection(lockCollectionName).deleteOne({ lock: 'lock' }));
+async function releaseLock(url: string, lockCollectionName: string, token: string): Promise<void> {
+  // Scoped to OUR token. An unscoped delete removes whatever lock exists — so after a missed
+  // heartbeat let another replica break and re-take the lock, the original holder's `finally`
+  // would delete the NEW holder's lock and let a third replica start a concurrent migration.
+  await dbRequest(url, (db) => db.collection(lockCollectionName).deleteOne({ lock: 'lock', owner: token }));
 }

package/src/core/modules/system-setup/README.md

@@ -164,7 +164,11 @@ NEST_SERVER_CONFIG='{ "systemSetup": { "initialAdmin": { "email": "admin@example
 - The admin is created automatically during application bootstrap (`OnApplicationBootstrap`)
 - Same zero-user guard applies: only works when no users exist
 - If users already exist, auto-creation is silently skipped (no error)
-- Race conditions between multiple instances are handled gracefully
+- Race conditions between multiple instances are handled gracefully: the zero-user check alone is
+  check-then-act, so N replicas booting at the same time would each pass it. Before creating, an
+  instance claims the setup by upserting the marker `{ _id: 'initial-admin' }` into the
+  `system-setup-locks` collection — an atomic operation only one instance wins. The others log a
+  debug message and skip. If creation fails, the marker is removed again so a later boot can retry.
 
 ### Security Best Practices
 
@@ -179,7 +183,8 @@ NEST_SERVER_CONFIG='{ "systemSetup": { "initialAdmin": { "email": "admin@example
 
 1. **Zero-user guard** - Init only works when `countDocuments({}) === 0`
 2. **Enabled by default** - Safe because endpoints are permanently locked once any user exists
-3. **Race condition protection** - MongoDB unique email index prevents duplicates
+3. **Race condition protection** - MongoDB unique email index prevents duplicates; auto-creation on
+   bootstrap is additionally claimed atomically via the `system-setup-locks` marker
 4. **Permanent lock** - Once any user exists, init returns 403
 5. **BetterAuth required** - Returns 403 if BetterAuth is not enabled
 

package/src/core/modules/system-setup/core-system-setup.service.ts

@@ -8,6 +8,25 @@ import { CoreBetterAuthUserMapper } from '../better-auth/core-better-auth-user.m
 import { CoreBetterAuthService } from '../better-auth/core-better-auth.service';
 import { ErrorCode } from '../error-code/error-codes';
 
+/**
+ * Collection holding the bootstrap claim markers.
+ *
+ * Native collection access is intentional here: no Mongoose schema exists for it
+ * (see docs/native-driver-security.md).
+ */
+const SETUP_LOCK_COLLECTION = 'system-setup-locks';
+
+/**
+ * How long an initial-admin claim may stand before another replica takes it over.
+ *
+ * Long enough that a slow but live creation is never stolen, short enough that a crashed
+ * claimer does not lock the deployment out for good.
+ */
+const INITIAL_ADMIN_CLAIM_STALE_AFTER_MS = 5 * 60_000;
+
+/** `_id` of the marker claiming the initial-admin creation */
+const INITIAL_ADMIN_LOCK_ID = 'initial-admin';
+
 /**
  * Input for creating the initial admin user
  */
@@ -101,6 +120,10 @@ export class CoreSystemSetupService implements OnApplicationBootstrap {
       return;
     }
 
+    // The atomic claim lives in createInitialAdmin() so that BOTH entry points — this
+    // one and the anonymous POST /system-setup/init — serialize on the same marker.
+    // Claiming here as well would take the claim twice on this path and leave the
+    // public path unguarded.
     try {
       const result = await this.createInitialAdmin({
         email: initialAdmin.email,
@@ -109,8 +132,11 @@ export class CoreSystemSetupService implements OnApplicationBootstrap {
       });
       this.logger.log(`Auto-created initial admin on startup: ${result.email}`);
     } catch (error) {
+      // The claim is released by createInitialAdmin() itself, which is the only place
+      // that takes it — so a replica that crashed mid-creation cannot block setup on
+      // every future boot, and the marker has exactly one owner.
       if (error instanceof ForbiddenException) {
-        this.logger.log('Initial admin auto-creation skipped (users already exist)');
+        this.logger.log('Initial admin auto-creation skipped (users already exist or claimed elsewhere)');
       } else {
         this.logger.warn(
           `Initial admin auto-creation failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
@@ -119,6 +145,58 @@ export class CoreSystemSetupService implements OnApplicationBootstrap {
     }
   }
 
+  /**
+   * Try to claim the initial-admin creation for this instance
+   *
+   * The upsert on a fixed `_id` is atomic, so exactly one of N concurrently booting
+   * replicas sees no previous document and wins the claim.
+   *
+   * @returns true when this instance may create the initial admin
+   */
+  protected async claimInitialAdminSetup(): Promise<boolean> {
+    try {
+      const collection = this.connection.collection(SETUP_LOCK_COLLECTION);
+
+      // The marker is a CLAIM, not a permanent record: a replica SIGKILLed between claiming and
+      // creating (OOM, node drain, a failed first rollout — all ordinary) never reaches the
+      // release in the catch below. Without an expiry that leaves a deployment with zero users
+      // and no way in, recoverable only by deleting a document from an undocumented collection.
+      // A stale claim is therefore taken over rather than obeyed.
+      const staleBefore = new Date(Date.now() - INITIAL_ADMIN_CLAIM_STALE_AFTER_MS);
+      await collection.deleteOne({ _id: INITIAL_ADMIN_LOCK_ID as any, claimedAt: { $lt: staleBefore } });
+
+      const previous = await collection.findOneAndUpdate(
+        { _id: INITIAL_ADMIN_LOCK_ID as any },
+        { $setOnInsert: { claimedAt: new Date() } },
+        { returnDocument: 'before', upsert: true },
+      );
+
+      // No previous document → this instance inserted the marker and owns the setup
+      return !previous;
+    } catch (error) {
+      // Two replicas upserting the same `_id` at the very same moment: one insert wins,
+      // the other gets a duplicate key error — which means the claim is taken.
+      if (error instanceof Error && (error.message?.includes('duplicate key') || error.message?.includes('E11000'))) {
+        return false;
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Remove the initial-admin claim marker so a later boot can retry the setup
+   */
+  protected async releaseInitialAdminSetupClaim(): Promise<void> {
+    try {
+      await this.connection.collection(SETUP_LOCK_COLLECTION).deleteOne({ _id: INITIAL_ADMIN_LOCK_ID as any });
+    } catch (error) {
+      // Never mask the failure that triggered the release
+      this.logger.warn(
+        `Failed to release initial admin setup claim: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      );
+    }
+  }
+
   /**
    * Check if the system needs initial setup (zero users)
    */
@@ -153,6 +231,17 @@ export class CoreSystemSetupService implements OnApplicationBootstrap {
       throw new ForbiddenException(ErrorCode.SYSTEM_SETUP_BETTERAUTH_REQUIRED);
     }
 
+    // The count check above is check-then-act, and this method is reachable ANONYMOUSLY
+    // via POST /system-setup/init. Two concurrent callers with DIFFERENT emails both pass
+    // it and both get an admin — the E11000 handler below only catches the same-email
+    // case. The claim is the same marker the boot path uses, so the HTTP path and the
+    // auto-bootstrap also serialize against each other instead of racing: an attacker
+    // racing a fresh deployment can no longer obtain an admin account ALONGSIDE the
+    // configured one.
+    if (!(await this.claimInitialAdminSetup())) {
+      throw new ForbiddenException(ErrorCode.SYSTEM_SETUP_NOT_AVAILABLE);
+    }
+
     try {
       // Access BetterAuth internal context (same pattern as core-better-auth-api.middleware.ts)
       const context = await authInstance.$context;
@@ -208,6 +297,11 @@ export class CoreSystemSetupService implements OnApplicationBootstrap {
         success: true,
       };
     } catch (error) {
+      // Release the claim we took above: a caller that failed mid-creation must not
+      // leave a deployment with zero users and no way in. The marker is additionally
+      // stale-expiring, so this is belt AND braces for the crash-before-release case.
+      await this.releaseInitialAdminSetupClaim();
+
       // Handle duplicate email (race condition via MongoDB unique index)
       if (error instanceof Error && (error.message?.includes('duplicate key') || error.message?.includes('E11000'))) {
         throw new ForbiddenException(ErrorCode.SYSTEM_SETUP_NOT_AVAILABLE);

package/src/core/modules/tenant/README.md

@@ -238,7 +238,7 @@ multiTenancy: {
 - **Config-change detection:** Cache is flushed when `multiTenancy` config changes (e.g., `roleHierarchy` update).
 - **Bounded:** Max 500 entries with FIFO eviction. Memory overhead: ~100-250 KB.
 
-**Important:** The cache is process-local. In horizontally scaled deployments (multiple instances), membership changes on one instance are not reflected on other instances until the TTL expires. Set `cacheTtlMs: 0` for security-sensitive deployments.
+**Important:** The cache storage itself is always process-local — every replica keeps its own map. What changed in 11.33.0 is how _invalidations_ travel: with `redis` configured they are broadcast, without it they are not (see below). A membership change that happens **outside** `invalidateUser()` / `invalidateAll()` — a direct DB write, an admin tool, a migration — is still invisible to every replica until `cacheTtlMs` expires. Set `cacheTtlMs: 0` for security-sensitive deployments.
 
 ### Manual Cache Invalidation
 
@@ -256,6 +256,26 @@ export class TenantService extends CoreTenantService {
 
 Use `invalidateAll()` to flush the entire cache (e.g., after bulk operations).
 
+Both are **instance** methods on the singleton `CoreTenantGuard` — inject it and call
+`this.tenantGuard?.invalidateUser(userId)`. There is no static equivalent.
+
+#### Cross-replica invalidation (since 11.33.0)
+
+`invalidateUser()` / `invalidateAll()` are no longer a purely local clear:
+
+| Setup                  | What happens                                                                                                                                                                                                                                       |
+| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **`redis` configured** | The local caches are cleared **and** the invalidation is published to `<keyPrefix>:tenant-cache:invalidate`. Every replica subscribed to that channel clears its own caches, so a membership change takes effect fleet-wide within one round trip. |
+| **No `redis`**         | Only the calling process clears. Other replicas keep serving the stale membership until `cacheTtlMs` expires — the pre-11.33.0 behaviour.                                                                                                          |
+
+Mechanism, for anyone extending the guard (`core-tenant.guard.ts`):
+
+- The channel is namespaced with the framework's Redis key prefix, so two applications sharing one Redis instance do not clear each other's caches. Two **stages** of the same application do, unless you set `redis.keyPrefix` or `redis.db`.
+- The message is `{ scope: 'user', userId }` or `{ scope: 'all' }`, JSON-encoded.
+- Subscription uses the shared subscriber connection from `CoreRedisService.getSubscriber()`; the listener is held in a field and detached on destroy, because that connection is shared.
+- **A received broadcast clears locally and does NOT re-publish** (`applyInvalidation()` → `clearUser()` / `clearAll()`, the no-broadcast variants). Re-publishing would bounce the message around the cluster forever.
+- **Publishing is fire-and-forget and failures are debug-level only.** A missed broadcast is not an error: it merely leaves the other replicas' caches stale until their TTL — exactly the no-Redis behaviour. Nothing about a request path depends on the broadcast succeeding.
+
 ### SkipTenantCheck
 
 For high-frequency endpoints that don't access tenant-scoped data, use `@SkipTenantCheck()` to avoid the membership lookup entirely.

package/src/core/modules/tenant/core-tenant.guard.ts

@@ -4,7 +4,9 @@ import {
   ForbiddenException,
   Injectable,
   Logger,
+  OnApplicationBootstrap,
   OnModuleDestroy,
+  Optional,
   UnauthorizedException,
 } from '@nestjs/common';
 import { Reflector } from '@nestjs/core';
@@ -14,6 +16,7 @@ import { Model } from 'mongoose';
 
 import { RoleEnum } from '../../common/enums/role.enum';
 import { ConfigService } from '../../common/services/config.service';
+import { CoreRedisService } from '../../common/services/core-redis.service';
 import { ErrorCode } from '../error-code/error-codes';
 import { CoreTenantMemberModel } from './core-tenant-member.model';
 import { SKIP_TENANT_CHECK_KEY } from './core-tenant.decorators';
@@ -42,6 +45,14 @@ interface CachedTenantIds {
   ids: string[];
 }
 
+/**
+ * Cache invalidation message broadcast between replicas via Redis pub/sub
+ */
+interface TenantCacheInvalidation {
+  scope: 'all' | 'user';
+  userId?: string;
+}
+
 /**
  * Global guard for multi-tenancy with defense-in-depth security.
  *
@@ -103,7 +114,7 @@ interface CachedTenantIds {
  *   - No user + checkable roles → 401 (authentication required)
  */
 @Injectable()
-export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
+export class CoreTenantGuard implements CanActivate, OnApplicationBootstrap, OnModuleDestroy {
   private readonly logger = new Logger(CoreTenantGuard.name);
 
   /**
@@ -111,6 +122,12 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
    * Key: `${userId}:${tenantId}`, Value: cached membership result with expiry.
    * Eliminates repeated DB queries for the same user+tenant combination.
    */
+  /** Channel this guard subscribed to, so it can unsubscribe again */
+  private invalidationChannel?: string;
+
+  /** The shared-subscriber listener, kept so it can be detached on destroy */
+  private invalidationListener?: (channel: string, message: string) => void;
+
   private readonly membershipCache = new Map<string, CachedMembership>();
 
   /**
@@ -131,6 +148,7 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
   constructor(
     private readonly reflector: Reflector,
     @InjectModel(TENANT_MEMBER_MODEL_TOKEN) private readonly memberModel: Model<CoreTenantMemberModel>,
+    @Optional() protected readonly redisService?: CoreRedisService,
   ) {
     // Clean up expired cache entries every 60 seconds
     this.cleanupInterval = setInterval(() => this.evictExpired(), 60_000);
@@ -139,7 +157,50 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
     }
   }
 
+  /**
+   * Subscribe to the cross-replica cache invalidation channel.
+   *
+   * Runs after ALL onModuleInit hooks so that CoreRedisService is connected.
+   * Without Redis this is a no-op and the caches stay process-local.
+   */
+  async onApplicationBootstrap(): Promise<void> {
+    if (!this.redisService?.enabled) {
+      return;
+    }
+    const channel = this.redisService.key('tenant-cache', 'invalidate');
+    try {
+      const subscriber = this.redisService.getSubscriber();
+      // Held in a field, not an anonymous closure: the subscriber connection is SHARED between
+      // features, so a listener that is never detached keeps dispatching into a destroyed
+      // guard's caches for the life of the process — and a second app in the same process
+      // inherits the previous one's listener alongside its own.
+      this.invalidationChannel = channel;
+      this.invalidationListener = (incomingChannel: string, message: string) => {
+        if (incomingChannel === channel) {
+          this.applyInvalidation(message);
+        }
+      };
+      subscriber.on('message', this.invalidationListener);
+      await subscriber.subscribe(channel);
+    } catch (error) {
+      this.logger.debug(`Tenant cache invalidation subscribe failed: ${(error as Error).message}`);
+    }
+  }
+
   onModuleDestroy(): void {
+    if (this.invalidationListener) {
+      try {
+        const subscriber = this.redisService?.getSubscriber();
+        subscriber?.off('message', this.invalidationListener);
+        if (this.invalidationChannel) {
+          void subscriber?.unsubscribe(this.invalidationChannel).catch(() => undefined);
+        }
+      } catch {
+        // Redis already gone — nothing to detach from
+      }
+      this.invalidationListener = undefined;
+    }
+
     if (this.cleanupInterval) {
       clearInterval(this.cleanupInterval);
       this.cleanupInterval = null;
@@ -152,31 +213,28 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
    * Invalidate all cache entries for a specific user.
    * Call this when memberships change (add/remove/update).
    *
+   * When Redis is enabled, the invalidation is additionally broadcast to all other
+   * replicas — otherwise their process-local caches would stay stale for up to cacheTtlMs.
+   *
    * Note: userId must not contain ':' characters (used as cache key delimiter).
    * MongoDB ObjectIds and standard UUID formats are safe.
    *
    * @param userId - The user ID whose cache entries should be invalidated
    */
   invalidateUser(userId: string): void {
-    for (const key of this.membershipCache.keys()) {
-      if (key.startsWith(`${userId}:`)) {
-        this.membershipCache.delete(key);
-      }
-    }
-    for (const key of this.tenantIdsCache.keys()) {
-      if (key === userId || key.startsWith(`${userId}:`)) {
-        this.tenantIdsCache.delete(key);
-      }
-    }
+    this.clearUser(userId);
+    this.publishInvalidation({ scope: 'user', userId });
   }
 
   /**
    * Clear all cache entries.
    * Useful when configuration changes (e.g., roleHierarchy) or for testing.
+   *
+   * When Redis is enabled, the invalidation is additionally broadcast to all other replicas.
    */
   invalidateAll(): void {
-    this.membershipCache.clear();
-    this.tenantIdsCache.clear();
+    this.clearAll();
+    this.publishInvalidation({ scope: 'all' });
   }
 
   async canActivate(context: ExecutionContext): Promise<boolean> {
@@ -192,7 +250,8 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
       const isTestEnv =
         process.env.VITEST === 'true' || process.env.NODE_ENV === 'test' || process.env.NODE_ENV === 'e2e';
       this.cacheTtlMs = config.cacheTtlMs ?? (isTestEnv ? 0 : 30_000);
-      this.invalidateAll();
+      // Local-only flush: every replica detects the config change itself, no broadcast needed
+      this.clearAll();
     }
 
     const request = this.getRequest(context);
@@ -494,6 +553,68 @@ export class CoreTenantGuard implements CanActivate, OnModuleDestroy {
   // Cache helpers
   // ===================================================================================================================
 
+  /**
+   * Apply an invalidation message received from another replica.
+   * Clears locally only — re-publishing would bounce the message around the cluster.
+   */
+  protected applyInvalidation(message: string): void {
+    let parsed: TenantCacheInvalidation;
+    try {
+      parsed = JSON.parse(message);
+    } catch {
+      this.logger.debug(`Ignoring malformed tenant cache invalidation message: ${message}`);
+      return;
+    }
+    if (parsed?.scope === 'all') {
+      this.clearAll();
+    } else if (parsed?.scope === 'user' && parsed.userId) {
+      this.clearUser(parsed.userId);
+    }
+  }
+
+  /**
+   * Clear both caches locally (no broadcast)
+   */
+  protected clearAll(): void {
+    this.membershipCache.clear();
+    this.tenantIdsCache.clear();
+  }
+
+  /**
+   * Remove all entries of a user from both caches locally (no broadcast)
+   */
+  protected clearUser(userId: string): void {
+    for (const key of this.membershipCache.keys()) {
+      if (key.startsWith(`${userId}:`)) {
+        this.membershipCache.delete(key);
+      }
+    }
+    for (const key of this.tenantIdsCache.keys()) {
+      if (key === userId || key.startsWith(`${userId}:`)) {
+        this.tenantIdsCache.delete(key);
+      }
+    }
+  }
+
+  /**
+   * Broadcast an invalidation to the other replicas (fire-and-forget).
+   * No-op without Redis; failures are logged at debug level only — a missed
+   * broadcast merely leaves the other replicas' caches stale until their TTL.
+   */
+  protected publishInvalidation(message: TenantCacheInvalidation): void {
+    if (!this.redisService?.enabled) {
+      return;
+    }
+    try {
+      void this.redisService
+        .getClient()
+        .publish(this.redisService.key('tenant-cache', 'invalidate'), JSON.stringify(message))
+        .catch((error: Error) => this.logger.debug(`Tenant cache invalidation publish failed: ${error.message}`));
+    } catch (error) {
+      this.logger.debug(`Tenant cache invalidation publish failed: ${(error as Error).message}`);
+    }
+  }
+
   /**
    * Look up a membership with process-level TTL cache.
    * Avoids repeated DB queries when the same user accesses the same tenant repeatedly.