@lensmcp/cluster 1.11.0 → 1.13.0

package/executors/gateway/runtime/lens-children.js

@@ -0,0 +1,136 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLensChildren = createLensChildren;
+/**
+ * Lens-mode managed children: the dashboard + MCP SINGLETONS (cluster-wide,
+ * owned by the gateway), and each `lens:true` app's Vite + browser capture.
+ * EVERY managed child AUTO-HEALS — restarted with exponential backoff when it
+ * exits while the gateway is up — and a dead child NEVER kills the cluster.
+ */
+const node_child_process_1 = require("node:child_process");
+const manifest_1 = require("../manifest");
+const lens_frontend_1 = require("@lensmcp/nx-plugin/lens-frontend");
+const scope_1 = require("./scope");
+const HEAL_BASE_MS = 1_500; // first restart delay
+const HEAL_MAX_MS = 30_000; // backoff cap — a true boot-loop settles here, never hot-loops
+const HEAL_HEALTHY_MS = 20_000; // ran at least this long ⇒ a FRESH crash ⇒ reset the backoff budget
+function createLensChildren(rt, obs, options) {
+    const { emit } = obs;
+    const lensChildren = [];
+    // Auto-heal bookkeeping per child label: consecutive restart attempts (drives the exponential
+    // backoff) so a boot-loop settles instead of hammering, reset once a child has run healthily.
+    const lensHeal = new Map();
+    // A `SpawnChild` over the gateway's managed-children array — the SAME shape `agent-dev` passes to
+    // `spawnLensFrontend`. EVERY gateway-managed child AUTO-HEALS: the dashboard + MCP singletons AND each
+    // `lens:true` vite app are restarted when they exit while the gateway is up, with exponential backoff.
+    // So a transient crash (a dev-server wedge, an OOM) recovers on its own — the user never hand-restarts —
+    // and a child that crashes-on-boot backs off to HEAL_MAX_MS rather than spinning. A child that ran
+    // healthily (≥ HEAL_HEALTHY_MS) gets a fresh budget, so one that dies after an hour comes back fast. A
+    // dead child still NEVER kills the cluster — it simply respawns in the background.
+    const spawnManagedChild = (label, bin, args, env, optional = false, respawn) => {
+        void optional;
+        void respawn; // legacy knobs — auto-heal is now universal (signature kept for compat)
+        const spawnedAt = Date.now();
+        const child = (0, node_child_process_1.spawn)(bin, args, { cwd: rt.root, env: { ...process.env, ...(env ?? {}) }, stdio: 'inherit' });
+        lensChildren.push(child);
+        child.on('exit', (code, sig) => {
+            const idx = lensChildren.indexOf(child);
+            if (idx >= 0)
+                lensChildren.splice(idx, 1); // drop the dead handle so stop() won't signal it
+            if (rt.stopped())
+                return; // the gateway is shutting down — do NOT resurrect
+            const heal = lensHeal.get(label) ?? { attempts: 0 };
+            if (Date.now() - spawnedAt >= HEAL_HEALTHY_MS)
+                heal.attempts = 0; // it was healthy → fresh budget
+            const delay = Math.min(HEAL_MAX_MS, HEAL_BASE_MS * 2 ** heal.attempts);
+            heal.attempts += 1;
+            lensHeal.set(label, heal);
+            console.warn(`[gateway] lens child ${label} exited (code=${code} sig=${sig}); auto-healing in ${Math.round(delay / 1000)}s (attempt ${heal.attempts}).`);
+            const t = setTimeout(() => { if (!rt.stopped())
+                spawnManagedChild(label, bin, args, env, optional, respawn); }, delay);
+            t.unref?.();
+        });
+        return child;
+    };
+    const bootSingletonsAndApps = () => {
+        const lensApps = [...new Map(rt.routes.filter((r) => r.lens).map((r) => [r.project, r.lens])).values()];
+        const wantDashboard = options.dashboard !== false;
+        const wantMcp = options.mcp !== false && lensApps.length > 0;
+        if (wantDashboard) {
+            const dashboardBundle = (0, lens_frontend_1.findDashboardBundle)(rt.root);
+            if (!dashboardBundle) {
+                console.warn("[gateway] lens dashboard bundle not found — skipping. Install the self-contained CLI: `npm i -D lensmcp` (it ships `bundled/dashboard.js`), or `yarn nx build @lensmcp/lensmcp-mcp` in-repo.");
+            }
+            else {
+                // Per-project lens scope: the dashboard is mounted at lensmcp.local/<key>/ so
+                // MANY projects coexist (each at its own /<key> on its own port). The dashboard
+                // OWNS its base path (LENSMCP_DASHBOARD_BASE) and history-routes under it, so the
+                // gateway routes pass-through (matchRoute on the prefix; the proxy never strips).
+                const scope = (0, scope_1.readLensScope)(rt.root);
+                const dashboardPort = options.dashboardPort ?? scope.dashboardPort;
+                const lensHost = options.lensHost ?? 'lensmcp.local';
+                const lensBase = scope.basePath; // e.g. /tetros
+                // Pushed BEFORE the cert SANs are derived (routes.map(r=>r.host)), so lensmcp.local
+                // auto-joins the TLS SANs. Run `gateway:trust` to add it to /etc/hosts.
+                rt.routes.push({ host: lensHost, prefix: lensBase, project: '(lens-dashboard)', target: `http://localhost:${dashboardPort}` });
+                (0, manifest_1.sortBySpecificity)(rt.routes);
+                console.log(`[gateway] starting lens dashboard → http://localhost:${dashboardPort} (routed at https://${lensHost}${lensBase}/)`);
+                emit('info', `lens dashboard up: ${lensHost}${lensBase} → :${dashboardPort}`, 'cluster-lens-dashboard', { kind: 'lens-dashboard-up', host: lensHost, port: dashboardPort, base: lensBase });
+                spawnManagedChild('lens-dashboard', process.execPath, [dashboardBundle], {
+                    LENSMCP_EVENT_FILE: rt.eventFile,
+                    LENSMCP_DASHBOARD_PORT: String(dashboardPort),
+                    LENSMCP_DASHBOARD_BASE: lensBase,
+                });
+            }
+        }
+        if (wantMcp) {
+            const mcpBundle = (0, lens_frontend_1.findMcpBundle)(rt.root);
+            if (!mcpBundle) {
+                console.warn("[gateway] MCP server bundle not found — skipping (lens apps will still publish to the bus; an external `lensmcp mcp` can read it). Install: `npm i -D lensmcp`.");
+            }
+            else {
+                console.log(`[gateway] starting lens MCP server (${mcpBundle})`);
+                spawnManagedChild('lens-mcp', process.execPath, [mcpBundle], {
+                    LENSMCP_EVENT_FILE: rt.eventFile,
+                    LENSMCP_TRANSPORT: process.env['LENSMCP_TRANSPORT'] ?? 'http',
+                    LENSMCP_PORT: process.env['LENSMCP_PORT'] ?? '3000',
+                });
+            }
+        }
+        // Spawn each `lens:true` frontend's Vite + browser capture as the upstream,
+        // via the SAME shared helper `agent-dev` uses (singletons excluded — owned above).
+        for (const app of lensApps) {
+            if (app.started)
+                continue;
+            app.started = true;
+            try {
+                (0, lens_frontend_1.spawnLensFrontend)({
+                    projectRoot: app.projectRoot,
+                    workspaceRoot: rt.root,
+                    project: app.project,
+                    eventFile: rt.eventFile,
+                    port: app.port,
+                    chrome: true,
+                    headless: true,
+                    // Identify this app's events on the shared bus (mirrors the serve
+                    // executor's LENSMCP_PROJECT stamp for pods).
+                    viteEnv: { LENSMCP_PROJECT: app.project },
+                    log: (l) => console.log(l),
+                }, spawnManagedChild);
+                emit('info', `lens app started: ${app.project} (vite :${app.port})`, `cluster-lens-app:${app.project}`, { kind: 'lens-app-up', project: app.project, port: app.port });
+            }
+            catch (e) {
+                console.error(`[gateway] lens app ${app.project}: ${e.message}`);
+            }
+        }
+    };
+    const reapAll = () => {
+        for (const c of lensChildren) {
+            try {
+                c.kill('SIGTERM');
+            }
+            catch { /* already gone */ }
+        }
+    };
+    return { spawnManagedChild, bootSingletonsAndApps, reapAll };
+}

package/executors/gateway/runtime/lifecycle.d.ts

@@ -0,0 +1,13 @@
+import type { GatewayRuntime, ServiceCtl, SockPool } from './types';
+import type { Observability } from './observability';
+export interface ServiceLayer {
+    scanNow(pool: SockPool): number;
+    spawnService(svc: ServiceCtl): void;
+    ensureUp(svc: ServiceCtl): Promise<boolean>;
+    scaleUp(svc: ServiceCtl): void;
+    killSpawned(svc: ServiceCtl, why: string): void;
+    /** Start the idle sweeper; returns the interval (already `.unref()`'d). */
+    startSweeper(): NodeJS.Timeout;
+}
+export declare function createServiceLayer(rt: GatewayRuntime, obs: Observability): ServiceLayer;
+//# sourceMappingURL=lifecycle.d.ts.map

package/executors/gateway/runtime/lifecycle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifecycle.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/lifecycle.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIrD,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAAC;IAChC,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,EAAE,UAAU,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;CAChC;AAED,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY,CAsIvF"}

package/executors/gateway/runtime/lifecycle.js

@@ -0,0 +1,179 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createServiceLayer = createServiceLayer;
+const tslib_1 = require("tslib");
+/**
+ * Service lifecycle: scale-from-zero, autoscale, idle kill. The gateway is the
+ * load balancer in front of each pod service's unix-socket pool and owns the
+ * pods' serverless lifecycle (eager/minPods/maxPods/idleKillSec).
+ */
+const node_child_process_1 = require("node:child_process");
+const fs = tslib_1.__importStar(require("node:fs"));
+const http = tslib_1.__importStar(require("node:http"));
+const path = tslib_1.__importStar(require("node:path"));
+const discovery_1 = require("./discovery");
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+function createServiceLayer(rt, obs) {
+    const { emit } = obs;
+    const scanNow = (pool) => { pool.scannedAt = 0; return (0, discovery_1.pickSock)(pool, rt.scanTtlMs) ? pool.socks.length : 0; };
+    const spawnService = (svc) => {
+        if (rt.stopped())
+            return; // never spawn a process after the gateway is told to stop
+        const target = `${svc.project}:${svc.decl.serveTarget ?? 'serve-hmr'}`;
+        const min = Math.max(1, svc.decl.minPods ?? 1);
+        console.log(`[gateway] starting ${target} (pods=${min})…`);
+        emit('info', `service starting: ${svc.project} (${min} pod${min > 1 ? 's' : ''})`, `cluster-service:${svc.project}`, { kind: 'service-starting', project: svc.project, pods: min });
+        svc.state = 'starting';
+        svc.spawnAt = Date.now();
+        // childCount goes as a CLI option, NOT env: nx executes tasks via its
+        // daemon, whose environment is not the gateway's — env vars get lost.
+        const proc = (0, node_child_process_1.spawn)('yarn', ['nx', 'run', target, `--childCount=${min}`], {
+            cwd: rt.root,
+            detached: true,
+            // NX_DAEMON=false keeps the serve task IN this spawned process group.
+            // With the daemon on, Nx hands the long-running task to the persistent
+            // daemon (a DIFFERENT group), so the later process-group kill reaps only
+            // the `yarn nx` shim and orphans the devserver + pod tree — the classic
+            // "idle-kill leaves live pods" leak. (--childCount as a CLI arg already
+            // avoids the env-loss the daemon was relied on for.)
+            env: { ...process.env, NX_DAEMON: 'false' },
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        svc.lastErrors = [];
+        svc.buildFailed = false;
+        const sniff = (d) => {
+            const text = d.toString();
+            for (const line of text.split('\n')) {
+                if (/ERROR in |error TS\d+|Found \d+ error|Unhandled 'error' event|EADDRINUSE/.test(line)) {
+                    // eslint-disable-next-line no-control-regex -- strip ANSI color codes from child output
+                    svc.lastErrors.push(line.replace(/\x1b\[[0-9;]*m/g, '').trim());
+                    if (svc.lastErrors.length > 12)
+                        svc.lastErrors.shift();
+                    svc.buildFailed = true;
+                    emit('error', `service error: ${svc.project}`, `cluster-service:${svc.project}`, { kind: 'service-error', project: svc.project, line: svc.lastErrors[svc.lastErrors.length - 1] });
+                }
+                if (/No typescript errors found|webpack compiled successfully|Child#\d+ ready/.test(line))
+                    svc.buildFailed = false;
+            }
+        };
+        proc.stdout?.on('data', (d) => { if (rt.stopped())
+            return; sniff(d); process.stdout.write(`[${svc.project}] ${d}`); });
+        proc.stderr?.on('data', (d) => { if (rt.stopped())
+            return; sniff(d); process.stderr.write(`[${svc.project}] ${d}`); });
+        proc.on('exit', (code) => {
+            svc.proc = undefined;
+            svc.state = 'down';
+            if (rt.stopped())
+                return; // gateway is tearing down — its dying services' exit chatter is noise
+            console.log(`[gateway] ${target} exited (${code})`);
+            emit(code === 0 ? 'info' : 'warning', `service down: ${svc.project} (exit ${code})`, `cluster-service:${svc.project}`, { kind: 'service-down', project: svc.project, code });
+        });
+        svc.proc = proc;
+    };
+    const ensureUp = async (svc) => {
+        if (scanNow(svc.pool) > 0) {
+            if (svc.state !== 'up') {
+                emit('info', `service up: ${svc.project} (${svc.pool.socks.length} pods, adopted)`, `cluster-service:${svc.project}`, { kind: 'service-up', project: svc.project, pods: svc.pool.socks.length, adopted: true });
+            }
+            svc.state = 'up';
+            return true;
+        }
+        if (rt.stopped())
+            return false;
+        if (svc.state !== 'starting' && !svc.proc)
+            spawnService(svc); // a live spawn supervises itself — don't stack a second one (EADDRINUSE)
+        const deadline = Date.now() + rt.startupTimeoutMs;
+        const failFastAt = Date.now() + 12_000; // a build error within 12s → stop waiting, report the cause
+        while (!rt.stopped() && Date.now() < deadline) {
+            await sleep(Math.min(700, rt.startupTimeoutMs / 4));
+            if (svc.buildFailed && Date.now() > failFastAt)
+                return false;
+            if (scanNow(svc.pool) > 0) {
+                svc.state = 'up';
+                svc.buildFailed = false;
+                const durationMs = svc.spawnAt ? Date.now() - svc.spawnAt : undefined;
+                console.log(`[gateway] ${svc.project} up (${svc.pool.socks.length} pods${durationMs ? `, cold start ${durationMs}ms` : ''})`);
+                emit('info', `cold start: ${svc.project} (${svc.pool.socks.length} pods, ${durationMs ?? '?'}ms)`, `cluster-service:${svc.project}`, { kind: 'cold-start', project: svc.project, pods: svc.pool.socks.length, durationMs });
+                return true;
+            }
+            if (svc.state === 'down')
+                return false; // spawn died
+        }
+        return false;
+    };
+    const scaleUp = (svc) => {
+        const max = svc.decl.maxPods ?? svc.decl.minPods ?? 1;
+        const cur = svc.pool.socks.length;
+        const admin = svc.decl.adminUrl ?? (svc.decl.port ? `http://localhost:${svc.decl.port}` : undefined);
+        if (!admin || cur === 0 || cur >= max || svc.inflight <= cur)
+            return;
+        if (Date.now() - svc.lastScaleAt < 5000)
+            return;
+        svc.lastScaleAt = Date.now();
+        const mod = admin.startsWith('https')
+            ? // eslint-disable-next-line @typescript-eslint/no-require-imports
+                require('node:https')
+            : http;
+        const r = mod.request(`${admin}/webpack/scale`, { method: 'POST', rejectUnauthorized: false }, (rs) => rs.resume());
+        r.on('error', () => undefined);
+        r.end(JSON.stringify({ pods: cur + 1 }));
+        console.log(`[gateway] autoscale ${svc.project}: ${cur} → ${cur + 1} pods (inflight ${svc.inflight})`);
+        emit('info', `autoscale: ${svc.project} ${cur} → ${cur + 1} pods`, `cluster-service:${svc.project}`, { kind: 'autoscale', project: svc.project, from: cur, to: cur + 1, inflight: svc.inflight });
+    };
+    // Idle sweeper: kill gateway-spawned lazy services after idleKillSec.
+    const killSpawned = (svc, why) => {
+        if (!svc.proc)
+            return;
+        console.log(`[gateway] ${svc.project} ${why} — scaling to zero.`);
+        emit('info', `idle kill: ${svc.project} (${why})`, `cluster-service:${svc.project}`, { kind: 'idle-kill', project: svc.project, why });
+        const dying = svc.proc;
+        // Detach the corpse's handlers BEFORE killing: a dying webpack/nx flushes
+        // buffered stdout/stderr as it exits, and those `sniff` callbacks would
+        // keep mutating svc.lastErrors/buildFailed (phantom build failures) — fatal
+        // if a fresh request re-spawns this same svc while the corpse still flushes.
+        try {
+            dying.stdout?.removeAllListeners('data');
+            dying.stderr?.removeAllListeners('data');
+            dying.removeAllListeners('exit');
+        }
+        catch { /* ignore */ }
+        try {
+            process.kill(-dying.pid, 'SIGTERM');
+        }
+        catch {
+            try {
+                dying.kill('SIGTERM');
+            }
+            catch { /* gone */ }
+        }
+        svc.proc = undefined;
+        svc.state = 'down';
+        // children die with the tree but their sock FILES survive — remove them
+        // so the canvas and the proxy never see ghost pods.
+        setTimeout(() => {
+            try {
+                for (const f of fs.readdirSync(svc.pool.dir)) {
+                    if (f.endsWith('.sock'))
+                        fs.unlinkSync(path.join(svc.pool.dir, f));
+                }
+            }
+            catch { /* gone */ }
+            svc.pool.socks = [];
+            svc.pool.scannedAt = 0;
+        }, 1200).unref?.();
+    };
+    const startSweeper = () => {
+        const sweeper = setInterval(() => {
+            for (const svc of rt.services) {
+                const idle = svc.decl.idleKillSec;
+                if (!svc.proc || !idle || svc.inflight > 0)
+                    continue;
+                if (Date.now() - svc.lastUsed > idle * 1000)
+                    killSpawned(svc, `idle ${idle}s`);
+            }
+        }, rt.sweepMs);
+        sweeper.unref();
+        return sweeper;
+    };
+    return { scanNow, spawnService, ensureUp, scaleUp, killSpawned, startSweeper };
+}

package/executors/gateway/runtime/observability.d.ts

@@ -0,0 +1,17 @@
+import type * as http from 'node:http';
+import type { GwTrace, Route } from './types';
+export type Severity = 'info' | 'warning' | 'error';
+export interface Observability {
+    emit(severity: Severity, title: string, fingerprint: string, raw?: Record<string, unknown>, flowCtx?: Record<string, unknown>): void;
+    eid(): string;
+    beginTrace(req: http.IncomingMessage): GwTrace | undefined;
+    traceStep(t: GwTrace | undefined, stepName: string, info?: Record<string, unknown>): void;
+    finishTrace(t: GwTrace | undefined, project: string, status: number, extra?: Record<string, unknown>): void;
+    recordEdge(route: Route, ms: number, status: number, caller?: string): void;
+    /** Start the periodic traffic-edge flush; returns the interval (already `.unref()`'d). */
+    startEdgeFlusher(): NodeJS.Timeout;
+}
+export declare function createObservability(eventFile: string, opts: {
+    trafficFlushMs: number;
+}): Observability;
+//# sourceMappingURL=observability.d.ts.map

package/executors/gateway/runtime/observability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"observability.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/observability.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AACvC,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9C,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACrI,GAAG,IAAI,MAAM,CAAC;IACd,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,GAAG,OAAO,GAAG,SAAS,CAAC;IAC3D,SAAS,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1F,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5G,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5E,0FAA0F;IAC1F,gBAAgB,IAAI,MAAM,CAAC,OAAO,CAAC;CACpC;AAKD,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CA6FtG"}

package/executors/gateway/runtime/observability.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createObservability = createObservability;
+const tslib_1 = require("tslib");
+/**
+ * The observability layer: everything the gateway writes to the LensMCP bus.
+ * Bus (`emit`/`eid`), per-request mechanism trace (`beginTrace`/`traceStep`/
+ * `finishTrace`), and aggregated host→service traffic edges (`recordEdge` +
+ * `startEdgeFlusher`). Merged because they only ever vary together — trace and
+ * edges both emit; both need `eid`.
+ *
+ * Appends BaseEvent-shaped lines to the shared event file the MCP tails (same
+ * rendezvous the vite plugin and nest instrumentation use), so the lens sees
+ * the cluster itself: services up/down, cold starts, autoscale, idle kills, and
+ * aggregated traffic edges for the topology.
+ */
+const fs = tslib_1.__importStar(require("node:fs"));
+const path = tslib_1.__importStar(require("node:path"));
+const hdr = (v) => typeof v === 'string' && v !== '' ? v : Array.isArray(v) ? v[0] : undefined;
+function createObservability(eventFile, opts) {
+    const eid = () => Date.now().toString(36) + Math.random().toString(36).slice(2, 12);
+    const emit = (severity, title, fingerprint, raw, flowCtx) => {
+        try {
+            fs.mkdirSync(path.dirname(eventFile), { recursive: true });
+            fs.appendFileSync(eventFile, JSON.stringify({
+                id: eid(), sessionId: 'pending', timestamp: Date.now(),
+                source: 'gateway', category: 'cluster', severity,
+                context: { sessionId: 'pending', ...flowCtx }, fingerprint, title, raw,
+            }) + '\n');
+        }
+        catch { /* lens offline — never block traffic */ }
+    };
+    // --- per-request mechanism trace (flows) ----------------------------------
+    // When a request carries lens flow headers (stamped by the browser client
+    // runtime or an instrumented upstream), the gateway narrates its own steps —
+    // received → route-match → auth → pod-select (+ cold-start) → response — as
+    // ONE `gateway-request` event correlated by the same flowId/requestId.
+    // Header-less traffic (curl, health checks) stays un-narrated so the bus
+    // doesn't bloat.
+    const beginTrace = (req) => {
+        const flowId = hdr(req.headers['x-lensmcp-flow-id']);
+        let requestId = hdr(req.headers['x-request-id']);
+        if (!flowId && !requestId && !hdr(req.headers['traceparent']))
+            return undefined;
+        if (!requestId) {
+            // Mint the request id AT THE EDGE and inject it downstream, so the
+            // gateway trace and the pod's server-request/spans share one id —
+            // that's the join key that attributes pod events to the routed service.
+            requestId = eid();
+            req.headers['x-request-id'] = requestId;
+        }
+        return {
+            startedAt: Date.now(),
+            method: (req.method ?? 'GET').toUpperCase(),
+            host: (req.headers.host || '').split(':')[0],
+            url: req.url ?? '/',
+            steps: [],
+            ...(flowId ? { flowId } : {}),
+            ...(requestId ? { requestId } : {}),
+            ...(hdr(req.headers['x-lensmcp-origin-node-id']) ? { originNodeId: hdr(req.headers['x-lensmcp-origin-node-id']) } : {}),
+        };
+    };
+    const traceStep = (t, stepName, info) => {
+        if (!t)
+            return;
+        t.steps.push({ step: stepName, atMs: Date.now() - t.startedAt, ...info });
+    };
+    const finishTrace = (t, project, status, extra) => {
+        if (!t || t.done)
+            return;
+        t.done = true;
+        const durationMs = Date.now() - t.startedAt;
+        emit(status >= 500 ? 'error' : 'info', `gateway ${t.method} ${t.host}${t.url} → ${status} (${durationMs}ms)`, `gateway-request:${project}`, {
+            kind: 'gateway-request', project, host: t.host, method: t.method, url: t.url,
+            status, durationMs, startedAt: t.startedAt, steps: t.steps, ...extra,
+        }, {
+            ...(t.flowId ? { flowId: t.flowId } : {}),
+            ...(t.requestId ? { requestId: t.requestId } : {}),
+            ...(t.originNodeId ? { originNodeId: t.originNodeId } : {}),
+        });
+    };
+    // --- aggregated host→service traffic edges --------------------------------
+    // The request-derived topology (nodes/edges appear from observed traffic).
+    const edges = new Map();
+    const recordEdge = (route, ms, status, caller) => {
+        const host = route.host ?? '(default)';
+        const key = (caller ? caller + '⇒' : '') + host + '→' + route.project;
+        const e = edges.get(key) ?? { host, project: route.project, caller, count: 0, totalMs: 0, errors: 0 };
+        e.count += 1;
+        e.totalMs += ms;
+        if (status >= 500 || status === 0)
+            e.errors += 1;
+        edges.set(key, e);
+    };
+    const startEdgeFlusher = () => {
+        const t = setInterval(() => {
+            for (const e of edges.values()) {
+                const avgMs = Math.round(e.totalMs / e.count);
+                emit(e.errors ? 'warning' : 'info', `traffic ${e.host} → ${e.project}: ${e.count} req, ~${avgMs}ms${e.errors ? `, ${e.errors} errors` : ''}`, `gateway-traffic:${e.host}->${e.project}`, { kind: 'gateway-traffic', host: e.host, project: e.project, caller: e.caller, count: e.count, avgMs, errors: e.errors });
+            }
+            edges.clear();
+        }, opts.trafficFlushMs);
+        t.unref();
+        return t;
+    };
+    return { emit, eid, beginTrace, traceStep, finishTrace, recordEdge, startEdgeFlusher };
+}

package/executors/gateway/runtime/proxy.d.ts

@@ -0,0 +1,20 @@
+import * as http from 'node:http';
+import type * as stream from 'node:stream';
+import type { GatewayRuntime, Route } from './types';
+import type { Observability } from './observability';
+import type { ServiceLayer } from './lifecycle';
+export interface ProxyServer {
+    web(req: http.IncomingMessage, res: http.ServerResponse, opts: Record<string, unknown>, cb: (err: Error) => void): void;
+    ws(req: http.IncomingMessage, socket: stream.Duplex, head: Buffer, opts: Record<string, unknown>, cb: (err?: Error) => void): void;
+    on(event: 'proxyRes', cb: (proxyRes: http.IncomingMessage, req: http.IncomingMessage) => void): void;
+    close?(): void;
+}
+export interface ProxyLayer {
+    proxy: ProxyServer;
+    forward(route: Route, req: http.IncomingMessage, res: http.ServerResponse): Promise<void>;
+    agentForTarget(target: unknown): http.Agent | import('node:https').Agent;
+    destroyAgents(): void;
+    closeProxy(): void;
+}
+export declare function createProxy(rt: GatewayRuntime, obs: Observability, svcLayer: ServiceLayer): ProxyLayer;
+//# sourceMappingURL=proxy.d.ts.map

package/executors/gateway/runtime/proxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/proxy.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,KAAK,KAAK,MAAM,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,EAAE,cAAc,EAAW,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAKhD,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACxH,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACnI,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,KAAK,CAAC,IAAI,IAAI,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,OAAO,YAAY,EAAE,KAAK,CAAC;IACzE,aAAa,IAAI,IAAI,CAAC;IACtB,UAAU,IAAI,IAAI,CAAC;CACpB;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,CA4FtG"}

package/executors/gateway/runtime/proxy.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createProxy = createProxy;
+const tslib_1 = require("tslib");
+/**
+ * The forwarding layer: one pooled `http-proxy` instance (keep-alive agents) and
+ * the `forward` that drives it — pod round-robin with scale-from-zero/autoscale,
+ * or a TCP upstream. Owns the edge-response branding (`x-lensmcp-pod`, stripping
+ * `x-powered-by`, `server: LensMCP`).
+ */
+const fs = tslib_1.__importStar(require("node:fs"));
+const http = tslib_1.__importStar(require("node:http"));
+const path = tslib_1.__importStar(require("node:path"));
+const discovery_1 = require("./discovery");
+const edge_1 = require("./edge");
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const httpProxy = require('http-proxy');
+function createProxy(rt, obs, svcLayer) {
+    const { recordEdge, finishTrace, traceStep } = obs;
+    // Keep-alive to upstreams (pods over unix sockets AND TCP apps). Without an
+    // agent http-proxy opens a fresh connection per request — a connect per call
+    // that ~halves throughput and adds latency. One pooled agent reuses sockets
+    // (keyed by socketPath / host:port); https targets get the verify-skipping
+    // https agent. NB: this file uses `require('node:https')` to match the source
+    // file's pattern (the orchestrator shadows the module with a local TLS flag).
+    const agentOpts = { keepAlive: true, keepAliveMsecs: 30_000, maxSockets: 1024, maxFreeSockets: 256 };
+    const httpAgent = new http.Agent(agentOpts);
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const httpsAgent = new (require('node:https').Agent)(agentOpts);
+    const agentForTarget = (target) => typeof target === 'string' && target.startsWith('https:') ? httpsAgent : httpAgent;
+    // secure:false — upstreams may terminate TLS with the same local CA.
+    // proxyTimeout — a pod that accepts the socket but never responds would
+    // otherwise pin svc.inflight forever (blocking idle-kill, skewing autoscale)
+    // and leave the gateway-request trace open; the timeout fires the proxy
+    // error path → 502 → res 'close' → inflight decrements + trace finishes.
+    const proxy = httpProxy.createProxyServer({ xfwd: true, secure: false, proxyTimeout: 120_000, agent: httpAgent });
+    // Surface WHICH pod served the request (round-robin made visible).
+    proxy.on('proxyRes', (proxyRes, req) => {
+        const pod = req.__lensmcpPod;
+        if (pod)
+            proxyRes.headers['x-lensmcp-pod'] = pod;
+        // The gateway owns the edge response: drop the upstream framework leak
+        // (e.g. NestJS/Express `x-powered-by`) and brand the hop as the LensMCP gateway.
+        delete proxyRes.headers['x-powered-by'];
+        proxyRes.headers['server'] = 'LensMCP';
+    });
+    const forward = async (route, req, res) => {
+        const startedAt = Date.now();
+        const trace = req.__lensmcpTrace;
+        res.on('close', () => {
+            recordEdge(route, Date.now() - startedAt, res.statusCode ?? 0, req.__lensmcpCaller);
+            finishTrace(trace, route.project, res.statusCode ?? 0, {
+                ...(req.__lensmcpPod ? { pod: req.__lensmcpPod } : {}),
+                ...(req.__lensmcpCaller ? { caller: req.__lensmcpCaller } : {}),
+            });
+        });
+        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix)) {
+            req.url = route.prependPrefix + (req.url ?? '/');
+        }
+        if (route.pool) {
+            const ctl = route.svc;
+            if (ctl) {
+                ctl.lastUsed = Date.now();
+                ctl.inflight += 1;
+                res.on('close', () => { ctl.inflight -= 1; ctl.lastUsed = Date.now(); });
+            }
+            let sock = (0, discovery_1.pickSock)(route.pool, rt.scanTtlMs);
+            if (!sock && ctl) {
+                // scale from zero: start the service, buffer this request until pods are up
+                traceStep(trace, 'cold-start', { project: route.project });
+                if (await svcLayer.ensureUp(ctl))
+                    sock = (0, discovery_1.pickSock)(route.pool, rt.scanTtlMs);
+            }
+            if (ctl)
+                svcLayer.scaleUp(ctl);
+            if (!sock) {
+                const errs = ctl?.lastErrors?.length ? ` Service errors: ${ctl.lastErrors.join('; ')}` : '';
+                (0, edge_1.sendError)(res, req, 503, 'unavailable', `Service ${route.project} is not reachable (no pods in ${route.pool.dir}).${errs} The watcher recovers and the next request retries automatically.`);
+                return;
+            }
+            req.__lensmcpPod = path.basename(sock, '.sock');
+            traceStep(trace, 'pod-select', { pod: path.basename(sock, '.sock'), pods: route.pool.socks.length, strategy: 'round-robin' });
+            // NO autoRewrite for a socketPath target: it's an OBJECT, and http-proxy's setRedirectHostRewrite
+            // does url.parse(options.target) on ANY 3xx response — which throws ("url must be a string, received
+            // Object") and CRASHES the gateway the instant a pod returns a redirect (e.g. the auth IdP's
+            // /authorize → login). A socket target has no host to rewrite anyway, and pods emit ABSOLUTE
+            // external redirect URLs, so no rewrite is needed.
+            proxy.web(req, res, { target: { socketPath: sock }, agent: httpAgent }, (err) => {
+                route.pool.socks = route.pool.socks.filter((s) => s !== sock); // drop dead pod until rescan
+                route.pool.idx = -1; // removing an element shifts indices — reset so the next pick walks cleanly from 0
+                try {
+                    fs.unlinkSync(sock);
+                }
+                catch { /* already gone */ } // a refused sock is a GHOST file — remove it so rescans stay truthful
+                console.error(`[gateway] pod ${path.basename(sock)} of ${route.project}:`, err.message);
+                (0, edge_1.sendError)(res, req, 502, 'unavailable', `Pod ${path.basename(sock)} of ${route.project} unavailable — retry.`);
+            });
+            return;
+        }
+        // autoRewrite ONLY for a string (external URL) target — a non-string target crashes
+        // setRedirectHostRewrite's url.parse(options.target) on a 3xx (see the socketPath case above).
+        proxy.web(req, res, { target: route.target, autoRewrite: typeof route.target === 'string', changeOrigin: true, agent: agentForTarget(route.target) }, (err) => {
+            console.error(`[gateway] upstream ${route.project} (${route.target}):`, err.message);
+            (0, edge_1.sendError)(res, req, 502, 'unavailable', `Upstream ${route.project} unavailable.`);
+        });
+    };
+    const destroyAgents = () => { httpAgent.destroy(); httpsAgent.destroy(); };
+    const closeProxy = () => { try {
+        proxy.close?.();
+    }
+    catch { /* ignore */ } };
+    return { proxy, forward, agentForTarget, destroyAgents, closeProxy };
+}

package/executors/gateway/runtime/scope.d.ts

@@ -0,0 +1,16 @@
+/** The base domain for the lens dashboard host — the longest common dotted
+ *  suffix of the declared cluster hosts (e.g. ['api.tetros.ai.local',
+ *  'tetros.ai.local'] → 'tetros.ai.local'). Falls back to the first host, then
+ *  to 'lens.local' if no host is declared. */
+export declare function baseDomainOf(hosts: string[]): string;
+/** Per-workspace lens scope (key + dashboard port + mount path). Prefers the
+ *  CLI-written `.lensmcp/config.json` (so the gateway agrees with `lensmcp gateway`
+ *  / `lensmcp dashboard`); falls back to deriving from the workspace. */
+export declare function readLensScope(root: string): {
+    key: string;
+    dashboardPort: number;
+    basePath: string;
+};
+export declare function lensKeyFrom(root: string): string;
+export declare function lensSlug(input: string): string;
+//# sourceMappingURL=scope.d.ts.map

package/executors/gateway/runtime/scope.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scope.d.ts","sourceRoot":"","sources":["../../../../../../libs/cluster/src/executors/gateway/runtime/scope.ts"],"names":[],"mappings":"AAOA;;;8CAG8C;AAC9C,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAUpD;AAED;;yEAEyE;AACzE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAmBpG;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhD;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQ9C"}