@lensmcp/cluster 1.16.31 → 1.17.0

package/executors/gateway/runtime/lens-children.d.ts

@@ -2,12 +2,34 @@ import { type SpawnChild } from '@lensmcp/nx-plugin/lens-frontend';
 import type { WorkspaceFragment } from './route-registry';
 import type { GatewayRuntime, GatewayRuntimeOptions } from './types';
 import { type Observability } from './observability';
-/** True when `port` can actually be BOUND on this machine right now. The in-process `used*Ports` sets only
- *  know what THIS daemon claimed — an ORPHANED child of a CRASHED daemon (or any foreign process) still
- *  holds its port at the OS level. Routing to such a port serves the WRONG workspace's content: the child
- *  we spawn crash-loops on EADDRINUSE while the route proxies into the stale squatter (the "tetros
- *  dashboard redirects to foodguard" bug). */
 export declare function probePortFree(port: number): Promise<boolean>;
+/** True when something is LISTENING on `port` on loopback — the liveness signal the exit handler cannot
+ *  give us ({@link LENS_PROBE_ENABLED}: a vite alive with zero LISTEN sockets).
+ *
+ *  BOTH loopback families are tried before reporting a failure. Vite binds the NAME `localhost`, which can
+ *  resolve to `::1` on a v6-preferring host — probing only `127.0.0.1` there would report every healthy dev
+ *  server as wedged and SIGKILL the whole cluster in a loop. The v6 attempt only runs on the v4 failure
+ *  path, so the healthy case stays one connect. */
+export declare function probePortListening(port: number, timeoutMs?: number): Promise<boolean>;
+/**
+ * Poll until `port` is actually BINDABLE, up to `timeoutMs`. Resolves `true` when it frees, `false` on
+ * timeout (the caller then spawns anyway and lets the normal heal handle the fallout).
+ *
+ * WHY a respawn must await this: a dying vite can hold its listener for seconds (open HMR websockets keep
+ * the server closing), so the fixed 1.5s heal delay raced it and the fresh child died on
+ * `listen EADDRINUSE` — a self-inflicted crash that burned a heal attempt and lengthened the outage
+ * (see {@link PORT_FREE_WAIT_MS}). Probes at least ONCE even with `timeoutMs: 0`, so the common already-free
+ * case costs a single probe and no delay.
+ */
+export declare function waitForPortFree(port: number, timeoutMs: number, deps?: {
+    isPortFree?: (port: number) => Promise<boolean>;
+    intervalMs?: number;
+}): Promise<boolean>;
+/** The `--port <n>` / `--port=<n>` a vite child was launched with. `spawnLensFrontend` always PINS the port
+ *  for a gateway-routed app (`--port <n> --strictPort`), so argv is the authoritative record of which port
+ *  this child owns — the port the liveness probe checks and the port its respawn must wait to be free.
+ *  `undefined` for an unpinned child (nothing to probe, nothing to wait for). */
+export declare function viteArgPort(args: readonly string[]): number | undefined;
 export interface LensChildren {
     spawnManagedChild: SpawnChild;
     /** Boot the dashboard + MCP singletons and every `lens:true` app. Pushes the

package/executors/gateway/runtime/lens-children.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AASA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;AAI1C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAuB,MAAM,SAAS,CAAC;AAQ1F,OAAO,EAA0B,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAM7E;;;;8CAI8C;AAC9C,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAO5D;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,EAAE,UAAU,CAAC;IAC9B;;gFAE4E;IAC5E,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC;;6GAEyG;IACzG,sBAAsB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACvF;;yGAEqG;IACrG,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACpE;gHAC4G;IAC5G,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,cAAc,EAClB,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,qBAAqB,EAE9B,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,GACzD,YAAY,CAmWd"}
+{"version":3,"file":"lens-children.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lens-children.ts"],"names":[],"mappings":"AASA,OAAO,EAIL,KAAK,UAAU,EAChB,MAAM,kCAAkC,CAAC;AAI1C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAuB,MAAM,SAAS,CAAC;AAgB1F,OAAO,EAA0B,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AA0B7E,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAalE;AAyBD;;;;;;mDAMmD;AACnD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAG1F;AAMD;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAClF,OAAO,CAAC,OAAO,CAAC,CAUlB;AAED;;;iFAGiF;AACjF,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAQvE;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,EAAE,UAAU,CAAC;IAC9B;;gFAE4E;IAC5E,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC;;6GAEyG;IACzG,sBAAsB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACvF;;yGAEqG;IACrG,iBAAiB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACpE;gHAC4G;IAC5G,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,cAAc,EAClB,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,qBAAqB,EAE9B,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,GACzD,YAAY,CAmbd"}

package/executors/gateway/runtime/lens-children.js

@@ -1,6 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.probePortFree = probePortFree;
+exports.probePortListening = probePortListening;
+exports.waitForPortFree = waitForPortFree;
+exports.viteArgPort = viteArgPort;
 exports.createLensChildren = createLensChildren;
 const tslib_1 = require("tslib");
 /**
@@ -27,14 +30,110 @@ const HEAL_HEALTHY_MS = 20_000; // ran at least this long ⇒ a FRESH crash ⇒
  *  holds its port at the OS level. Routing to such a port serves the WRONG workspace's content: the child
  *  we spawn crash-loops on EADDRINUSE while the route proxies into the stale squatter (the "tetros
  *  dashboard redirects to foodguard" bug). */
-function probePortFree(port) {
+/** One bind attempt. `in-use` only for the errnos that actually mean "someone has it"; any other failure
+ *  means that ADDRESS FAMILY is unavailable on this machine (no IPv6, for instance), which is not evidence
+ *  either way and must never be reported as "taken" — that would make `claimFreePort` step up forever. */
+function tryBind(port, host) {
     return new Promise((resolve) => {
         const probe = net.createServer();
         probe.unref();
-        probe.once('error', () => resolve(false));
-        probe.listen(port, () => probe.close(() => resolve(true)));
+        probe.once('error', (e) => {
+            const code = e.code ?? '';
+            resolve(code === 'EADDRINUSE' || code === 'EACCES' ? 'in-use' : 'unavailable');
+        });
+        probe.listen(host === undefined ? { port } : { port, host }, () => probe.close(() => resolve('free')));
+    });
+}
+async function probePortFree(port) {
+    // The WILDCARD bind alone is NOT enough, and that hole defeated the very purpose of this probe. MEASURED:
+    // with a holder on `127.0.0.1:P`, `listen({ port: P })` (the wildcard) reports **FREE** — a wildcard bind
+    // does not collide with a specific-address bind on macOS/BSD. Our children bind the NAME `localhost`
+    // (vite `--port N --strictPort`, the dashboard bundles), i.e. a LOOPBACK address — exactly the case the
+    // wildcard probe cannot see. So an orphaned vite holding `127.0.0.1:4200` looked free, `claimFreePort`
+    // handed 4200 straight back out, and the fresh child died on `listen EADDRINUSE` — the self-inflicted
+    // crash this probe exists to prevent (and what makes `waitForPortFree` meaningful rather than a no-op).
+    // Free means bindable on the wildcard AND on both loopback addresses; unavailable families are skipped.
+    for (const host of [undefined, '127.0.0.1', '::1']) {
+        if ((await tryBind(port, host)) === 'in-use')
+            return false;
+    }
+    return true;
+}
+/** A single-address TCP connect: does `host:port` ACCEPT right now? Connect-and-destroy, never an HTTP
+ *  request — a WEDGED vite may accept then hang forever, so a `GET` would time out even where the listener
+ *  is fine (and would pollute the dev server's log). Every timer/socket is unref'd: a probe must never hold
+ *  the gateway process open. */
+function probeConnect(host, port, timeoutMs) {
+    return new Promise((resolve) => {
+        let done = false;
+        const sock = net.createConnection({ host, port });
+        sock.unref();
+        const timer = setTimeout(() => finish(false), timeoutMs);
+        timer.unref?.();
+        function finish(ok) {
+            if (done)
+                return;
+            done = true;
+            clearTimeout(timer);
+            sock.destroy();
+            resolve(ok);
+        }
+        sock.once('connect', () => finish(true));
+        sock.once('error', () => finish(false));
     });
 }
+/** True when something is LISTENING on `port` on loopback — the liveness signal the exit handler cannot
+ *  give us ({@link LENS_PROBE_ENABLED}: a vite alive with zero LISTEN sockets).
+ *
+ *  BOTH loopback families are tried before reporting a failure. Vite binds the NAME `localhost`, which can
+ *  resolve to `::1` on a v6-preferring host — probing only `127.0.0.1` there would report every healthy dev
+ *  server as wedged and SIGKILL the whole cluster in a loop. The v6 attempt only runs on the v4 failure
+ *  path, so the healthy case stays one connect. */
+async function probePortListening(port, timeoutMs = 1_000) {
+    if (await probeConnect('127.0.0.1', port, timeoutMs))
+        return true;
+    return probeConnect('::1', port, timeoutMs);
+}
+/** Unref'd sleep — a heal/backoff wait must never keep the gateway alive past `stop()`. */
+const sleepUnref = (ms) => new Promise((r) => { const t = setTimeout(r, ms); t.unref?.(); });
+/**
+ * Poll until `port` is actually BINDABLE, up to `timeoutMs`. Resolves `true` when it frees, `false` on
+ * timeout (the caller then spawns anyway and lets the normal heal handle the fallout).
+ *
+ * WHY a respawn must await this: a dying vite can hold its listener for seconds (open HMR websockets keep
+ * the server closing), so the fixed 1.5s heal delay raced it and the fresh child died on
+ * `listen EADDRINUSE` — a self-inflicted crash that burned a heal attempt and lengthened the outage
+ * (see {@link PORT_FREE_WAIT_MS}). Probes at least ONCE even with `timeoutMs: 0`, so the common already-free
+ * case costs a single probe and no delay.
+ */
+async function waitForPortFree(port, timeoutMs, deps = {}) {
+    const isFree = deps.isPortFree ?? probePortFree;
+    const intervalMs = Math.max(1, deps.intervalMs ?? types_1.PORT_FREE_POLL_MS);
+    const deadline = Date.now() + Math.max(0, timeoutMs);
+    for (;;) {
+        if (await isFree(port))
+            return true;
+        const left = deadline - Date.now();
+        if (left <= 0)
+            return false;
+        await sleepUnref(Math.min(intervalMs, left));
+    }
+}
+/** The `--port <n>` / `--port=<n>` a vite child was launched with. `spawnLensFrontend` always PINS the port
+ *  for a gateway-routed app (`--port <n> --strictPort`), so argv is the authoritative record of which port
+ *  this child owns — the port the liveness probe checks and the port its respawn must wait to be free.
+ *  `undefined` for an unpinned child (nothing to probe, nothing to wait for). */
+function viteArgPort(args) {
+    const valid = (n) => (Number.isInteger(n) && n > 0 && n < 65_536 ? n : undefined);
+    for (let i = 0; i < args.length; i += 1) {
+        const a = args[i];
+        if (a === '--port')
+            return valid(Number(args[i + 1]));
+        if (a.startsWith('--port='))
+            return valid(Number(a.slice('--port='.length)));
+    }
+    return undefined;
+}
 function createLensChildren(rt, obs, options, 
 // Injectable OS-level port probe (unit tests stub it; real ports are machine-dependent).
 deps) {
@@ -43,10 +142,14 @@ deps) {
     const lensChildren = [];
     // Auto-heal bookkeeping per child label: consecutive restart attempts (drives the exponential
     // backoff) so a boot-loop settles instead of hammering, reset once a child has run healthily.
+    // `portConflicts` counts CONSECUTIVE lost bind races (EADDRINUSE) — forgiven rather than charged to
+    // `attempts`, but capped so a permanently-squatted port still falls back to backoff.
     const lensHeal = new Map();
-    // Zombie recycle: the live `lens:true` app VITE children (label 'vite') + when each spawned, so the lens
-    // sweeper can restart one whose fs-watcher missed a post-spawn file add (the reported dashboard zombie).
-    // The capture sidecar + the dashboard/MCP singletons are NOT tracked (they don't serve app source).
+    // Zombie recycle + LIVENESS probe: the live `lens:true` app VITE children (label 'vite') + when each
+    // spawned, its pinned `port`, and its run of consecutive probe failures. `port`/`probeFails` drive the
+    // wedge probe (alive but NOT listening); `spawnedAt` also drives the stale-source-set recycle. Both reset
+    // naturally on a respawn, which registers a FRESH handle. The capture sidecar + the dashboard/MCP
+    // singletons are NOT tracked (they don't serve app source).
     const viteChildren = new Map();
     // Per-project last-recycle time — the cooldown survives the kill→auto-heal→respawn (a new child handle).
     const viteRecycleAt = new Map();
@@ -101,9 +204,14 @@ deps) {
         lensChildren.push(child);
         labeledChildren.set(label, child); // the CURRENT handle for this label (a per-workspace dashboard reap targets it)
         // Track a `lens:true` app's vite dev server (label 'vite', or `<ws>:<proj>:vite` for a registered
-        // workspace) for the zombie recycle. Its `LENSMCP_PROJECT` names the app; a fresh respawn re-adds here.
-        if (label === 'vite' || label.endsWith(':vite'))
-            viteChildren.set(child, { project: env?.['LENSMCP_PROJECT'] ?? 'app', spawnedAt });
+        // workspace) for the zombie recycle + the liveness probe. Its `LENSMCP_PROJECT` names the app and its
+        // argv pins the port; a fresh respawn re-adds here with a clean probe run.
+        const isVite = label === 'vite' || label.endsWith(':vite');
+        // The port THIS child owns (pinned `--port <n> --strictPort`): what the liveness probe checks and what
+        // the respawn must wait to be free. `undefined` for the dashboard/MCP singletons + an unpinned vite.
+        const ownPort = isVite ? viteArgPort(args) : undefined;
+        if (isVite)
+            viteChildren.set(child, { project: env?.['LENSMCP_PROJECT'] ?? 'app', spawnedAt, port: ownPort, probeFails: 0 });
         child.on('exit', (code, sig) => {
             const idx = lensChildren.indexOf(child);
             if (idx >= 0)
@@ -111,37 +219,126 @@ deps) {
             viteChildren.delete(child); // the auto-heal respawn re-registers a fresh handle with a new spawnedAt
             if (rt.stopped() || removedLabels.has(label))
                 return; // shutting down, or this workspace unregistered — do NOT resurrect
-            const heal = lensHeal.get(label) ?? { attempts: 0 };
-            if (Date.now() - spawnedAt >= HEAL_HEALTHY_MS)
-                heal.attempts = 0; // it was healthy → fresh budget
-            const delay = Math.min(HEAL_MAX_MS, HEAL_BASE_MS * 2 ** heal.attempts);
-            heal.attempts += 1;
-            lensHeal.set(label, heal);
-            console.warn(`[gateway] lens child ${label} exited (code=${code} sig=${sig}); auto-healing in ${Math.round(delay / 1000)}s (attempt ${heal.attempts}).`);
-            const t = setTimeout(() => { if (!rt.stopped())
-                spawnManagedChild(label, bin, args, env, optional, respawn, cwd); }, delay);
-            t.unref?.();
+            const ranMs = Date.now() - spawnedAt;
+            const heal = lensHeal.get(label) ?? { attempts: 0, portConflicts: 0 };
+            // The heal is ASYNC from here: classifying a bind race and waiting for the port both need a probe.
+            // Every wait is unref'd and every resume re-checks stopped()/removedLabels, so shutdown still wins.
+            void (async () => {
+                // BIND RACE vs CRASH. A fast non-zero exit while ANOTHER process still holds our pinned port is
+                // vite's `--strictPort` losing an EADDRINUSE race with the child it replaced — an environment
+                // condition, not a code fault. Charging it to `attempts` is what pushed the observed heal to
+                // "attempt 2" and doubled the outage; forgive it (bounded) and wait the port out instead.
+                const portHeld = ownPort !== undefined && !(await isPortFree(ownPort));
+                const raceLost = (0, lifecycle_1.isPortConflictExit)({ code, signal: sig, ranMs, portHeld }, { fastFailMs: types_1.CHILD_START_FAIL_MS });
+                if (raceLost && heal.portConflicts < types_1.MAX_PORT_CONFLICT_RETRIES) {
+                    heal.portConflicts += 1; // `attempts` UNTOUCHED — the crash-loop budget is for real crashes
+                    lensHeal.set(label, heal);
+                    console.warn(`[gateway] lens child ${label} lost the :${ownPort} bind race (EADDRINUSE, code=${code}); waiting for the port, then retrying (port-retry ${heal.portConflicts}/${types_1.MAX_PORT_CONFLICT_RETRIES}; crash budget untouched).`);
+                }
+                else {
+                    heal.portConflicts = 0; // a real exit ends the bind-race run
+                    if (ranMs >= HEAL_HEALTHY_MS)
+                        heal.attempts = 0; // it was healthy → fresh budget
+                    const delay = Math.min(HEAL_MAX_MS, HEAL_BASE_MS * 2 ** heal.attempts);
+                    heal.attempts += 1;
+                    lensHeal.set(label, heal);
+                    console.warn(`[gateway] lens child ${label} exited (code=${code} sig=${sig}); auto-healing in ${Math.round(delay / 1000)}s (attempt ${heal.attempts}).`);
+                    await sleepUnref(delay);
+                }
+                if (rt.stopped() || removedLabels.has(label))
+                    return;
+                // NEVER respawn into the dying child's port. A closing vite can hold its listener for seconds (open
+                // HMR websockets), so the old fixed delay raced it → `listen EADDRINUSE` → a self-inflicted crash
+                // that burned a heal attempt and LENGTHENED the 502. Bounded: past the budget we spawn anyway and
+                // let the (now correctly-classified) heal take it from there.
+                if (ownPort !== undefined && !(await waitForPortFree(ownPort, types_1.PORT_FREE_WAIT_MS, { isPortFree, intervalMs: types_1.PORT_FREE_POLL_MS }))) {
+                    console.warn(`[gateway] lens child ${label}: :${ownPort} still held after ${Math.round(types_1.PORT_FREE_WAIT_MS / 1000)}s — spawning anyway.`);
+                }
+                if (rt.stopped() || removedLabels.has(label))
+                    return;
+                spawnManagedChild(label, bin, args, env, optional, respawn, cwd);
+            })().catch(() => undefined); // a heal must never surface as an unhandled rejection
         });
         return child;
     };
-    // ZOMBIE recycle for lens frontends: restart a `lens:true` app's vite whose fs-watcher went stale — it is
-    // ALIVE + reachable yet MISSED files created after it spawned (500 `Failed to resolve import` for a file
-    // that exists; the reported 17.7h dashboard pod). The shared `rt.sourceSetChangedAt` (stamped by the
-    // lifecycle sweeper's source-set scan) drives it: a vite that spawned before a settled set change is
-    // killed → this manager's own `on('exit')` auto-heal respawns a fresh vite that re-scans the FS. Only on a
-    // set CHANGE (add/remove), never a content save, and never a blind max-age (no idle signal for a frontend).
+    // LIVENESS probe for lens frontends: kill a vite that is ALIVE but NOT LISTENING (the permanent-502 wedge
+    // — see LENS_PROBE_ENABLED). One cheap TCP connect per tracked child per tick; the pure `lensChildWedged`
+    // owns the verdict (grace + N consecutive failures + the shared per-project cooldown), this owns the I/O.
+    // A confirmed wedge gets SIGKILL — deliberately NOT SIGTERM: graceful shutdown is exactly what wedged (the
+    // event loop is pinned by the lens WS server / bridge sockets), so a SIGTERM can hang and leave the app
+    // 502ing forever. The kill lands on the proven exit→auto-heal path, which respawns after waiting the port
+    // out. `probeInFlight` keeps a slow round from overlapping the next tick.
+    let probeInFlight = false;
+    const probeLensChildren = async (cfg) => {
+        if (probeInFlight)
+            return;
+        probeInFlight = true;
+        try {
+            for (const [child, meta] of [...viteChildren]) {
+                if (rt.stopped())
+                    return;
+                if (meta.port === undefined)
+                    continue; // unpinned port ⇒ nothing to probe
+                if (child.exitCode !== null || child.signalCode !== null)
+                    continue; // already dying — the exit heal owns it
+                if (Date.now() - meta.spawnedAt < cfg.graceMs) {
+                    meta.probeFails = 0;
+                    continue;
+                } // still booting
+                if (await probePortListening(meta.port, types_1.LENS_PROBE_TIMEOUT_MS)) {
+                    meta.probeFails = 0;
+                    continue;
+                }
+                meta.probeFails += 1;
+                const reason = (0, lifecycle_1.lensChildWedged)(meta, viteRecycleAt.get(meta.project) ?? 0, Date.now(), cfg);
+                if (!reason)
+                    continue; // failing, but not yet N-in-a-row / still in cooldown
+                viteRecycleAt.set(meta.project, Date.now());
+                meta.probeFails = 0;
+                console.error(`[gateway] wedge recycle: lens app ${meta.project} is ALIVE but NOT LISTENING on :${meta.port} after ${Math.max(2, cfg.failureThreshold)} probes — SIGKILL; a fresh vite is coming.`);
+                emit('warning', `wedge recycle: ${meta.project} (${reason})`, `cluster-lens-app:${meta.project}`, { kind: 'pod-recycle', project: meta.project, reason, port: meta.port });
+                try {
+                    child.kill('SIGKILL');
+                }
+                catch { /* already gone — the exit auto-heal will respawn */ }
+            }
+        }
+        finally {
+            probeInFlight = false;
+        }
+    };
+    // The lens-frontend sweeper drives BOTH self-heals over the tracked vite children:
+    //  1. the LIVENESS probe above — alive but not listening (a permanent 502, invisible to the exit handler);
+    //  2. ZOMBIE recycle — a vite whose fs-watcher went stale: ALIVE + reachable yet MISSING files created
+    //     after it spawned (500 `Failed to resolve import` for a file that exists; the reported 17.7h
+    //     dashboard pod). The shared `rt.sourceSetChangedAt` (stamped by the lifecycle sweeper's source-set
+    //     scan) drives it: a vite that spawned before a settled set change is killed → this manager's own
+    //     `on('exit')` auto-heal respawns a fresh vite that re-scans the FS. Only on a set CHANGE
+    //     (add/remove), never a content save, and never a blind max-age (no idle signal for a frontend).
     const startLensSweeper = () => {
-        if (!types_1.POD_RECYCLE_ENABLED || lensSweeper)
+        // Either heal alone justifies the sweeper: a user who disabled the staleness recycle (its churn was a
+        // known problem) must still get a wedged 502 healed, so each branch is gated on its OWN switch.
+        if (lensSweeper || (!types_1.POD_RECYCLE_ENABLED && !types_1.LENS_PROBE_ENABLED))
             return;
         const cfg = { graceMs: types_1.POD_STALE_GRACE_MS, settleMs: types_1.POD_RECYCLE_SETTLE_MS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS };
+        const probeCfg = { graceMs: types_1.LENS_PROBE_GRACE_MS, failureThreshold: types_1.LENS_PROBE_FAILS, cooldownMs: types_1.POD_STALE_RECYCLE_COOLDOWN_MS };
         lensSweeper = setInterval(() => {
-            if (rt.stopped())
+            if (rt.stopped() || viteChildren.size === 0)
+                return;
+            // The liveness probe is INDEPENDENT of the source-set clock — a wedge has nothing to do with the file
+            // set, so it must run before (and regardless of) the staleness early-return below.
+            if (types_1.LENS_PROBE_ENABLED)
+                void probeLensChildren(probeCfg);
+            if (!types_1.POD_RECYCLE_ENABLED)
                 return;
-            const changedAt = rt.sourceSetChangedAt ?? 0;
-            if (changedAt <= 0 || viteChildren.size === 0)
+            const wideChangedAt = rt.sourceSetChangedAt ?? 0;
+            if (wideChangedAt <= 0)
                 return;
             const now = Date.now();
             for (const [child, meta] of viteChildren) {
+                // Prefer this app's SCOPED clock — the file set within the subtrees ITS vite can resolve. On the wide
+                // clock a backend-only `.ts` add anywhere in the monorepo restarted every frontend (`sourceScopeDirs`).
+                const changedAt = rt.sourceSetChangedFor?.get(meta.project) ?? wideChangedAt;
                 const last = viteRecycleAt.get(meta.project) ?? 0;
                 if (!(0, lifecycle_1.lensFrontendStale)(meta.spawnedAt, last, changedAt, now, cfg))
                     continue;

package/executors/gateway/runtime/lifecycle.d.ts

@@ -3,8 +3,34 @@ import type { Observability } from './observability';
 /** Bounded, async, non-blocking walk of the workspace source tree → a signature of the file SET
  *  (`<count>:<xor-of-path-hashes>`). Skips node_modules/dist/build/dotdirs + `*-devserver` sock dirs + every
  *  GITIGNORED path (runtime output is not source — see `SRC_EXCLUDE`), and only counts source-extension
- *  files. Order-independent (readdir order irrelevant) and null-safe. */
-export declare function sourceSetSignature(root: string): Promise<string>;
+ *  files. Order-independent (readdir order irrelevant) and null-safe.
+ *
+ *  `scopeDirs` (absolute, from {@link sourceScopeDirs}) narrows the walk to the subtrees a given child could
+ *  actually resolve, instead of the whole monorepo — see that function for WHY. Omitted ⇒ the whole root
+ *  (the original workspace-wide behavior, still used when a child's project root is unknown). Root-level
+ *  loose files are always counted, so a `package.json`/`tsconfig.base.json` add reaches every scope. */
+export declare function sourceSetSignature(root: string, scopeDirs?: readonly string[], extraExclude?: ReadonlySet<string>): Promise<string>;
+/**
+ * PURE: which subtrees a child of `projectRoot` must watch for a "file set changed" verdict.
+ *
+ * The staleness signal exists to catch a vite/pod whose fs-watcher MISSED a file add. But the signature was
+ * workspace-WIDE while its consumers are PER-CHILD, so adding a backend-only `server/apps/auth/**\/*.ts`
+ * — a file in no vite module graph — recycled every frontend vite too. Worked failure (foodguard,
+ * 2026-07-28): one session writing `server/apps/auth/src/modules/totp/**` added a `.ts` every ~15-20s and
+ * held `app.foodguard.local` in near-continuous 502s, 951 recycles in a session, none of them relevant to
+ * the dashboard.
+ *
+ * The scope is the child's OWN top-level bucket plus every bucket that hosts no OTHER gateway-managed
+ * project (`managedBuckets`). In a scope-first monorepo (`web/ server/ shared/ ai/ tools/`) that yields
+ * `web + shared + ai + tools` for a frontend and `server + shared + ai + tools` for a service: a
+ * `server/**` add no longer touches a frontend, while a `shared/contracts` add still reaches both — which
+ * is correct, since both import it. It deliberately OVER-includes (a frontend watches `ai/` it may never
+ * import) because a false recycle costs a cold start whereas a MISSED one costs a wedged dev server.
+ *
+ * Both args are absolute; the result is absolute and deduped. An empty result (the project root IS the
+ * workspace root, or it sits outside it) means "no narrowing" → the caller falls back to the whole root.
+ */
+export declare function sourceScopeDirs(workspaceRoot: string, projectRoot: string, managedBuckets: readonly string[], topLevelDirs: readonly string[], extraExclude?: ReadonlySet<string>): string[];
 export interface PodRecycleConfig {
     maxAgeMs: number;
     graceMs: number;
@@ -32,6 +58,59 @@ export declare function lensFrontendStale(spawnedAt: number, lastRecycleAt: numb
     settleMs: number;
     cooldownMs: number;
 }): boolean;
+/** Config for the lens-child LIVENESS decision ({@link lensChildWedged}). */
+export interface LensChildProbeConfig {
+    /** Spawn grace: no verdict until the child has had this long to BIND its port (a cold vite boot). */
+    graceMs: number;
+    /** CONSECUTIVE probe failures required. Floored at 2 — one blip must never recycle. */
+    failureThreshold: number;
+    /** Per-project anti-thrash window after any recycle (shared with {@link lensFrontendStale}). */
+    cooldownMs: number;
+}
+/**
+ * PURE decision: is this ALIVE lens-frontend child WEDGED — process running, port NOT listening?
+ *
+ * The third self-heal blind spot. `lensFrontendStale` catches a vite serving a stale FS view; the exit
+ * handler catches a vite that DIED. Neither sees a vite that is alive, has printed `VITE ready`, and has
+ * NO LISTEN socket — its event loop held open by the lens WS server / bridge sockets while its HTTP
+ * listener is gone. Nothing exits, so nothing auto-heals, and the app 502s until a human kills the pid
+ * (foodguard 2026-07-29 — see {@link LENS_PROBE_ENABLED}).
+ *
+ * The caller owns the socket I/O (a cheap TCP connect per tick) and accumulates `probeFails`; this owns
+ * WHEN that evidence justifies a kill. Three guards, each earning its place:
+ *   - **grace** — a still-booting vite has no listener YET; killing it would be the outage, not the fix.
+ *   - **N consecutive failures** (≥2, enforced here so even a misconfigured `failureThreshold: 1` can't
+ *     single-blip recycle) — a momentary refusal is not a wedge.
+ *   - **cooldown** — the same per-project window the two recycle paths share, so probe + staleness can
+ *     never thrash one project between them.
+ * Returns the recycle reason or `null`. Kept pure (no process, no socket) so all four are unit-tested.
+ */
+export declare function lensChildWedged(child: {
+    spawnedAt: number;
+    probeFails: number;
+}, lastRecycleAt: number, now: number, cfg: LensChildProbeConfig): 'wedged-no-listener' | null;
+/**
+ * PURE decision: was this child's exit a lost BIND RACE (EADDRINUSE) rather than a crash?
+ *
+ * The auto-heal respawned on a fixed backoff while the child it replaced still held the pinned port, so
+ * vite's `--strictPort` exited code=1 on bind — and that self-inflicted exit consumed a heal attempt,
+ * pushing the backoff up and lengthening the very 502 it was healing (observed: `listen EADDRINUSE
+ * 127.0.0.1:50059` → `exited (code=1 sig=null); auto-healing in 3s (attempt 2)`). A bind race is an
+ * ENVIRONMENT condition, not a code fault, so it must not spend the crash-loop budget.
+ *
+ * The signature, from evidence we already have (stdio is `inherit`, so the child's stderr is unreadable):
+ * a non-zero exit, NOT from a signal (we killed it), FAST enough that it cannot have served traffic, while
+ * some OTHER process still holds the port. If the port is free by now the next spawn will simply succeed,
+ * so it is treated as a normal heal — the conservative direction.
+ */
+export declare function isPortConflictExit(exit: {
+    code: number | null;
+    signal: NodeJS.Signals | string | null;
+    ranMs: number;
+    portHeld: boolean;
+}, cfg: {
+    fastFailMs: number;
+}): boolean;
 export interface ServiceLayer {
     scanNow(pool: SockPool): number;
     spawnService(svc: ServiceCtl): void;

package/executors/gateway/runtime/lifecycle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAapE,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,iBAAiB,CAAC;AAyE/D;;;yEAGyE;AACzE,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA6BtE;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC,EACpG,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,gBAAgB,GACpB,kBAAkB,GAAG,SAAS,GAAG,IAAI,CAQvC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAMT;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CAiYvF"}
+{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAapE,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,iBAAiB,CAAC;AAyE/D;;;;;;;;wGAQwG;AACxG,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,YAAY,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAqCzI;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,eAAe,CAC7B,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,YAAY,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,GACjC,MAAM,EAAE,CAcV;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC,EACpG,GAAG,EAAE,MAAM,EACX,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,gBAAgB,GACpB,kBAAkB,GAAG,SAAS,GAAG,IAAI,CAQvC;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,MAAM,EAC1B,GAAG,EAAE,MAAM,EACX,GAAG,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAMT;AAED,6EAA6E;AAC7E,MAAM,WAAW,oBAAoB;IACnC,qGAAqG;IACrG,OAAO,EAAE,MAAM,CAAC;IAChB,uFAAuF;IACvF,gBAAgB,EAAE,MAAM,CAAC;IACzB,gGAAgG;IAChG,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,EAChD,aAAa,EAAE,MAAM,EACrB,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,oBAAoB,GACxB,oBAAoB,GAAG,IAAI,CAK7B;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,EACvG,GAAG,EAAE;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,GAC1B,OAAO,CAKT;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CAmcvF"}