@lensmcp/cluster 1.0.0

package/executors/gateway/gateway.lib.js

@@ -0,0 +1,797 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hostMatches = void 0;
+exports.verifyDevJwt = verifyDevJwt;
+exports.stampIdentity = stampIdentity;
+exports.pickSock = pickSock;
+exports.discoverRoutes = discoverRoutes;
+exports.startGateway = startGateway;
+const tslib_1 = require("tslib");
+/**
+ * The gateway runtime, separated from the Nx executor so it is directly
+ * testable: `startGateway()` returns a handle with the actually-bound ports
+ * and a `stop()` — no signals needed. The executor is a thin wrapper.
+ */
+const node_child_process_1 = require("node:child_process");
+const node_crypto_1 = require("node:crypto");
+const fs = tslib_1.__importStar(require("node:fs"));
+const http = tslib_1.__importStar(require("node:http"));
+const os = tslib_1.__importStar(require("node:os"));
+const path = tslib_1.__importStar(require("node:path"));
+const manifest_1 = require("./manifest");
+var manifest_2 = require("./manifest"); // back-compat re-export
+Object.defineProperty(exports, "hostMatches", { enumerable: true, get: function () { return manifest_2.hostMatches; } });
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const httpProxy = require('http-proxy');
+/**
+ * Dev edge auth (HS256), mirroring the prod gateway: verify the end-user token, then stamp the
+ * trusted identity headers the services read (via GatewayTrustGuard). RS256/JWKS is prod's job;
+ * dev shares the `AUTH_JWT_SECRET` the auth host signs with. Returns the claims, or undefined.
+ */
+function verifyDevJwt(req) {
+    const secret = process.env['AUTH_JWT_SECRET'] ?? 'tetros-dev-secret';
+    const auth = String((Array.isArray(req.headers['authorization']) ? req.headers['authorization'][0] : req.headers['authorization']) ?? '');
+    if (!auth.startsWith('Bearer '))
+        return undefined;
+    const [h, p, s] = auth.slice(7).split('.');
+    if (!h || !p || !s)
+        return undefined;
+    const expected = (0, node_crypto_1.createHmac)('sha256', secret).update(`${h}.${p}`).digest('base64url');
+    const a = Buffer.from(s);
+    const b = Buffer.from(expected);
+    if (a.length !== b.length || !(0, node_crypto_1.timingSafeEqual)(a, b))
+        return undefined;
+    try {
+        const claims = JSON.parse(Buffer.from(p, 'base64url').toString());
+        if (typeof claims['exp'] === 'number' && Date.now() / 1000 > claims['exp'])
+            return undefined;
+        return claims;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Stamp verified identity onto the forwarded request — tenant from the `tid` CLAIM, never a client header. */
+function stampIdentity(req, claims) {
+    const set = (k, v) => { if (v !== undefined && v !== null && v !== '')
+        req.headers[k] = String(v); };
+    set('x-user-id', claims['sub']);
+    set('x-tenant-id', claims['tid']);
+    const roles = claims['roles'];
+    if (Array.isArray(roles) && roles.length)
+        req.headers['x-user-roles'] = roles.map(String).join(',');
+    const perms = claims['perms'];
+    if (Array.isArray(perms) && perms.length)
+        req.headers['x-user-permissions'] = perms.map(String).join(',');
+}
+const devAttrs = (req, claims) => ({
+    ...claims,
+    ip: req.socket.remoteAddress ?? '',
+    method: (req.method ?? 'GET').toUpperCase(),
+    path: (req.url ?? '/').split('?')[0],
+});
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Round-robin over the pool's sockets, rescanning the dir when the cache ages out. */
+function pickSock(pool, scanTtlMs = 1500) {
+    const now = Date.now();
+    if (now - pool.scannedAt > scanTtlMs) {
+        try {
+            pool.socks = fs.readdirSync(pool.dir)
+                .filter((f) => /^child-\d+\.sock$/.test(f))
+                .map((f) => path.join(pool.dir, f));
+        }
+        catch {
+            pool.socks = [];
+        }
+        pool.scannedAt = now;
+    }
+    if (pool.socks.length === 0)
+        return undefined;
+    pool.idx = (pool.idx + 1) % pool.socks.length;
+    return pool.socks[pool.idx];
+}
+/**
+ * Read every project's `cluster` (or legacy `davnx`) declaration and build
+ * the route table: host routes first (plus auto `internal.<host>` for pod
+ * services), the `default` catch-all last.
+ */
+function discoverRoutes(rootDir, projects, sockBaseDir = os.tmpdir()) {
+    const routes = [];
+    const services = [];
+    let defaultRoute;
+    for (const [name, p] of Object.entries(projects)) {
+        let decl;
+        try {
+            const raw = JSON.parse(fs.readFileSync(path.join(rootDir, p.root, 'project.json'), 'utf8'));
+            decl = raw.cluster ?? raw.davnx;
+        }
+        catch {
+            continue;
+        }
+        if (!decl)
+            continue;
+        let pool;
+        let svc;
+        let target;
+        if (decl.service) {
+            pool = { dir: path.join(sockBaseDir, `${decl.service}-devserver`), socks: [], idx: -1, scannedAt: 0 };
+            svc = { project: name, decl, pool, state: 'down', lastUsed: Date.now(), inflight: 0, lastScaleAt: 0 };
+            services.push(svc);
+        }
+        else {
+            target = decl.upstream ?? (decl.port ? `http://localhost:${decl.port}` : undefined);
+            if (!target) {
+                console.warn(`[gateway] ${name}: no service/upstream/port — skipped.`);
+                continue;
+            }
+        }
+        const base = { project: name, prependPrefix: decl.prependPrefix, target, pool, svc, prefix: decl.path, auth: decl.auth };
+        if (decl.host) {
+            routes.push({ ...base, host: decl.host });
+            // internal.<host>: east-west traffic, same pods, NO middleware.
+            if (pool && decl.internalHost !== false) {
+                routes.push({
+                    ...base,
+                    host: typeof decl.internalHost === 'string' ? decl.internalHost : `internal.${decl.host}`,
+                    internal: true,
+                });
+            }
+        }
+        if (decl.default)
+            defaultRoute = { ...base, prefix: undefined };
+    }
+    if (defaultRoute)
+        routes.push(defaultRoute);
+    // specificity sort (shared with prod): host+path beats host-only beats
+    // catch-all — so a path-mounted service (api.x/channels) wins over the
+    // host's owner (api.x → api).
+    (0, manifest_1.sortBySpecificity)(routes);
+    return { routes, services };
+}
+async function startGateway(options, context) {
+    const https = options.https !== false;
+    const ports = options.ports?.length ? options.ports : [443];
+    const scanTtlMs = options.scanTtlMs ?? 1500;
+    const startupTimeoutMs = options.startupTimeoutMs ?? 120_000;
+    // Set by stop(); consulted by spawnService/ensureUp so an in-flight cold
+    // start neither spawns a process nor logs after the gateway is torn down.
+    let stopped = false;
+    // --- LensMCP bus: the gateway is a first-class lens source -------------------
+    // Appends BaseEvent-shaped lines to the shared event file the MCP tails
+    // (same rendezvous the vite plugin and nest instrumentation use), so the
+    // lens sees the cluster itself: services up/down, cold starts, autoscale,
+    // idle kills, and aggregated host→service traffic edges for the topology.
+    const eventFile = process.env['LENSMCP_EVENT_FILE'] ?? path.join(context.root, '.lensmcp', 'events.jsonl');
+    const eid = () => Date.now().toString(36) + Math.random().toString(36).slice(2, 12);
+    const emit = (severity, title, fingerprint, raw, flowCtx) => {
+        try {
+            fs.mkdirSync(path.dirname(eventFile), { recursive: true });
+            fs.appendFileSync(eventFile, JSON.stringify({
+                id: eid(), sessionId: 'pending', timestamp: Date.now(),
+                source: 'gateway', category: 'cluster', severity,
+                context: { sessionId: 'pending', ...flowCtx }, fingerprint, title, raw,
+            }) + '\n');
+        }
+        catch { /* lens offline — never block traffic */ }
+    };
+    const hdr = (v) => typeof v === 'string' && v !== '' ? v : Array.isArray(v) ? v[0] : undefined;
+    const beginTrace = (req) => {
+        const flowId = hdr(req.headers['x-lensmcp-flow-id']);
+        let requestId = hdr(req.headers['x-request-id']);
+        if (!flowId && !requestId && !hdr(req.headers['traceparent']))
+            return undefined;
+        if (!requestId) {
+            // Mint the request id AT THE EDGE and inject it downstream, so the
+            // gateway trace and the pod's server-request/spans share one id —
+            // that's the join key that attributes pod events to the routed service.
+            requestId = eid();
+            req.headers['x-request-id'] = requestId;
+        }
+        return {
+            startedAt: Date.now(),
+            method: (req.method ?? 'GET').toUpperCase(),
+            host: (req.headers.host || '').split(':')[0],
+            url: req.url ?? '/',
+            steps: [],
+            ...(flowId ? { flowId } : {}),
+            ...(requestId ? { requestId } : {}),
+            ...(hdr(req.headers['x-lensmcp-origin-node-id']) ? { originNodeId: hdr(req.headers['x-lensmcp-origin-node-id']) } : {}),
+        };
+    };
+    const traceStep = (t, stepName, info) => {
+        if (!t)
+            return;
+        t.steps.push({ step: stepName, atMs: Date.now() - t.startedAt, ...info });
+    };
+    const finishTrace = (t, project, status, extra) => {
+        if (!t || t.done)
+            return;
+        t.done = true;
+        const durationMs = Date.now() - t.startedAt;
+        emit(status >= 500 ? 'error' : 'info', `gateway ${t.method} ${t.host}${t.url} → ${status} (${durationMs}ms)`, `gateway-request:${project}`, {
+            kind: 'gateway-request', project, host: t.host, method: t.method, url: t.url,
+            status, durationMs, startedAt: t.startedAt, steps: t.steps, ...extra,
+        }, {
+            ...(t.flowId ? { flowId: t.flowId } : {}),
+            ...(t.requestId ? { requestId: t.requestId } : {}),
+            ...(t.originNodeId ? { originNodeId: t.originNodeId } : {}),
+        });
+    };
+    // --- discover ---------------------------------------------------------------
+    const { routes, services } = discoverRoutes(context.root, context.projectsConfigurations?.projects ?? {});
+    if (routes.length === 0) {
+        throw new Error('[gateway] no project declares a `cluster` field — nothing to route.');
+    }
+    // --- service lifecycle: scale-from-zero, autoscale, idle kill ----------------
+    const scanNow = (pool) => { pool.scannedAt = 0; return pickSock(pool, scanTtlMs) ? pool.socks.length : 0; };
+    const spawnService = (svc) => {
+        if (stopped)
+            return; // never spawn a process after the gateway is told to stop
+        const target = `${svc.project}:${svc.decl.serveTarget ?? 'serve-hmr'}`;
+        const min = Math.max(1, svc.decl.minPods ?? 1);
+        console.log(`[gateway] starting ${target} (pods=${min})…`);
+        emit('info', `service starting: ${svc.project} (${min} pod${min > 1 ? 's' : ''})`, `cluster-service:${svc.project}`, { kind: 'service-starting', project: svc.project, pods: min });
+        svc.state = 'starting';
+        svc.spawnAt = Date.now();
+        // childCount goes as a CLI option, NOT env: nx executes tasks via its
+        // daemon, whose environment is not the gateway's — env vars get lost.
+        const proc = (0, node_child_process_1.spawn)('yarn', ['nx', 'run', target, `--childCount=${min}`], {
+            cwd: context.root,
+            detached: true,
+            // NX_DAEMON=false keeps the serve task IN this spawned process group.
+            // With the daemon on, Nx hands the long-running task to the persistent
+            // daemon (a DIFFERENT group), so the later process-group kill reaps only
+            // the `yarn nx` shim and orphans the devserver + pod tree — the classic
+            // "idle-kill leaves live pods" leak. (--childCount as a CLI arg already
+            // avoids the env-loss the daemon was relied on for.)
+            env: { ...process.env, NX_DAEMON: 'false' },
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        svc.lastErrors = [];
+        svc.buildFailed = false;
+        const sniff = (d) => {
+            const text = d.toString();
+            for (const line of text.split('\n')) {
+                if (/ERROR in |error TS\d+|Found \d+ error|Unhandled 'error' event|EADDRINUSE/.test(line)) {
+                    svc.lastErrors.push(line.replace(/\x1b\[[0-9;]*m/g, '').trim());
+                    if (svc.lastErrors.length > 12)
+                        svc.lastErrors.shift();
+                    svc.buildFailed = true;
+                    emit('error', `service error: ${svc.project}`, `cluster-service:${svc.project}`, { kind: 'service-error', project: svc.project, line: svc.lastErrors[svc.lastErrors.length - 1] });
+                }
+                if (/No typescript errors found|webpack compiled successfully|Child#\d+ ready/.test(line))
+                    svc.buildFailed = false;
+            }
+        };
+        proc.stdout?.on('data', (d) => { if (stopped)
+            return; sniff(d); process.stdout.write(`[${svc.project}] ${d}`); });
+        proc.stderr?.on('data', (d) => { if (stopped)
+            return; sniff(d); process.stderr.write(`[${svc.project}] ${d}`); });
+        proc.on('exit', (code) => {
+            svc.proc = undefined;
+            svc.state = 'down';
+            if (stopped)
+                return; // gateway is tearing down — its dying services' exit chatter is noise
+            console.log(`[gateway] ${target} exited (${code})`);
+            emit(code === 0 ? 'info' : 'warning', `service down: ${svc.project} (exit ${code})`, `cluster-service:${svc.project}`, { kind: 'service-down', project: svc.project, code });
+        });
+        svc.proc = proc;
+    };
+    const ensureUp = async (svc) => {
+        if (scanNow(svc.pool) > 0) {
+            if (svc.state !== 'up') {
+                emit('info', `service up: ${svc.project} (${svc.pool.socks.length} pods, adopted)`, `cluster-service:${svc.project}`, { kind: 'service-up', project: svc.project, pods: svc.pool.socks.length, adopted: true });
+            }
+            svc.state = 'up';
+            return true;
+        }
+        if (stopped)
+            return false;
+        if (svc.state !== 'starting' && !svc.proc)
+            spawnService(svc); // a live spawn supervises itself — don't stack a second one (EADDRINUSE)
+        const deadline = Date.now() + startupTimeoutMs;
+        const failFastAt = Date.now() + 12_000; // a build error within 12s → stop waiting, report the cause
+        while (!stopped && Date.now() < deadline) {
+            await sleep(Math.min(700, startupTimeoutMs / 4));
+            if (svc.buildFailed && Date.now() > failFastAt)
+                return false;
+            if (scanNow(svc.pool) > 0) {
+                svc.state = 'up';
+                svc.buildFailed = false;
+                const durationMs = svc.spawnAt ? Date.now() - svc.spawnAt : undefined;
+                console.log(`[gateway] ${svc.project} up (${svc.pool.socks.length} pods${durationMs ? `, cold start ${durationMs}ms` : ''})`);
+                emit('info', `cold start: ${svc.project} (${svc.pool.socks.length} pods, ${durationMs ?? '?'}ms)`, `cluster-service:${svc.project}`, { kind: 'cold-start', project: svc.project, pods: svc.pool.socks.length, durationMs });
+                return true;
+            }
+            if (svc.state === 'down')
+                return false; // spawn died
+        }
+        return false;
+    };
+    const scaleUp = (svc) => {
+        const max = svc.decl.maxPods ?? svc.decl.minPods ?? 1;
+        const cur = svc.pool.socks.length;
+        const admin = svc.decl.adminUrl ?? (svc.decl.port ? `http://localhost:${svc.decl.port}` : undefined);
+        if (!admin || cur === 0 || cur >= max || svc.inflight <= cur)
+            return;
+        if (Date.now() - svc.lastScaleAt < 5000)
+            return;
+        svc.lastScaleAt = Date.now();
+        const mod = admin.startsWith('https')
+            ? // eslint-disable-next-line @typescript-eslint/no-require-imports
+                require('node:https')
+            : http;
+        const r = mod.request(`${admin}/webpack/scale`, { method: 'POST', rejectUnauthorized: false }, (rs) => rs.resume());
+        r.on('error', () => undefined);
+        r.end(JSON.stringify({ pods: cur + 1 }));
+        console.log(`[gateway] autoscale ${svc.project}: ${cur} → ${cur + 1} pods (inflight ${svc.inflight})`);
+        emit('info', `autoscale: ${svc.project} ${cur} → ${cur + 1} pods`, `cluster-service:${svc.project}`, { kind: 'autoscale', project: svc.project, from: cur, to: cur + 1, inflight: svc.inflight });
+    };
+    // Idle sweeper: kill gateway-spawned lazy services after idleKillSec.
+    const killSpawned = (svc, why) => {
+        if (!svc.proc)
+            return;
+        console.log(`[gateway] ${svc.project} ${why} — scaling to zero.`);
+        emit('info', `idle kill: ${svc.project} (${why})`, `cluster-service:${svc.project}`, { kind: 'idle-kill', project: svc.project, why });
+        const dying = svc.proc;
+        // Detach the corpse's handlers BEFORE killing: a dying webpack/nx flushes
+        // buffered stdout/stderr as it exits, and those `sniff` callbacks would
+        // keep mutating svc.lastErrors/buildFailed (phantom build failures) — fatal
+        // if a fresh request re-spawns this same svc while the corpse still flushes.
+        try {
+            dying.stdout?.removeAllListeners('data');
+            dying.stderr?.removeAllListeners('data');
+            dying.removeAllListeners('exit');
+        }
+        catch { /* ignore */ }
+        try {
+            process.kill(-dying.pid, 'SIGTERM');
+        }
+        catch {
+            try {
+                dying.kill('SIGTERM');
+            }
+            catch { /* gone */ }
+        }
+        svc.proc = undefined;
+        svc.state = 'down';
+        // children die with the tree but their sock FILES survive — remove them
+        // so the canvas and the proxy never see ghost pods.
+        setTimeout(() => {
+            try {
+                for (const f of fs.readdirSync(svc.pool.dir)) {
+                    if (f.endsWith('.sock'))
+                        fs.unlinkSync(path.join(svc.pool.dir, f));
+                }
+            }
+            catch { /* gone */ }
+            svc.pool.socks = [];
+            svc.pool.scannedAt = 0;
+        }, 1200).unref?.();
+    };
+    const sweeper = setInterval(() => {
+        for (const svc of services) {
+            const idle = svc.decl.idleKillSec;
+            if (!svc.proc || !idle || svc.inflight > 0)
+                continue;
+            if (Date.now() - svc.lastUsed > idle * 1000)
+                killSpawned(svc, `idle ${idle}s`);
+        }
+    }, options.sweepMs ?? 10_000);
+    sweeper.unref();
+    // Eager services boot with the gateway.
+    for (const svc of services) {
+        if (svc.decl.eager)
+            void ensureUp(svc);
+    }
+    // --- per-service api keys (be-security: the gateway is the trust boundary) --
+    // Every pod service gets a stable key in {cacheDir}/service-keys.json.
+    // Public routes: the gateway injects the TARGET's key as x-internal-token —
+    // pods are only reachable through it (unix sockets aren't on the network).
+    // internal.<host> routes: the CALLER must present its own key as x-api-key;
+    // the gateway validates it, stamps x-api-key-id (caller identity, for audit
+    // and lens traces) and forwards with the target's x-internal-token.
+    const keysDir = path.join(context.root, 'node_modules', '.cache', 'davnx-webpack');
+    const keysPath = path.join(keysDir, 'service-keys.json');
+    let serviceKeys = {};
+    try {
+        serviceKeys = JSON.parse(fs.readFileSync(keysPath, 'utf8'));
+    }
+    catch { /* first run */ }
+    let keysDirty = false;
+    for (const svc of services) {
+        if (!serviceKeys[svc.project]) {
+            serviceKeys[svc.project] = (0, node_crypto_1.randomBytes)(16).toString('hex');
+            keysDirty = true;
+        }
+    }
+    if (keysDirty) {
+        try {
+            fs.mkdirSync(keysDir, { recursive: true });
+            fs.writeFileSync(keysPath, JSON.stringify(serviceKeys, null, 2));
+        }
+        catch { /* ro fs */ }
+    }
+    const keyToProject = new Map(Object.entries(serviceKeys).map(([p, k]) => [k, p]));
+    // --- middleware (the JWT seam; internal.* routes skip it) -------------------
+    let middleware;
+    if (options.middleware) {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const mod = require(path.resolve(context.root, options.middleware));
+        middleware = (mod.default ?? mod);
+    }
+    // --- routing ------------------------------------------------------------------
+    // The matcher is SHARED with the production gateway (one routing brain, no
+    // drift — see manifest.ts `matchRoute` for the rules). Dev's Route carries
+    // pod/svc fields too; matchRoute only reads host/prefix.
+    const match = (req) => (0, manifest_1.matchRoute)(routes, (req.headers.host || '').split(':')[0], req.url ?? '/');
+    // secure:false — upstreams may terminate TLS with the same local CA.
+    // proxyTimeout — a pod that accepts the socket but never responds would
+    // otherwise pin svc.inflight forever (blocking idle-kill, skewing autoscale)
+    // and leave the gateway-request trace open; the timeout fires the proxy
+    // error path → 502 → res 'close' → inflight decrements + trace finishes.
+    // Keep-alive to upstreams (pods over unix sockets AND TCP apps). Without an
+    // agent http-proxy opens a fresh connection per request — a connect per call
+    // that ~halves throughput and adds latency. One pooled agent reuses sockets
+    // (keyed by socketPath / host:port); https targets get the verify-skipping
+    // https agent. Picked per request by target in `forward`/`upgrade`.
+    // NB: a local `const https` (TLS-on flag) shadows the module here, so the
+    // https agent is built via require — matching this file's node:https usage.
+    const agentOpts = { keepAlive: true, keepAliveMsecs: 30_000, maxSockets: 1024, maxFreeSockets: 256 };
+    const httpAgent = new http.Agent(agentOpts);
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const httpsAgent = new (require('node:https').Agent)(agentOpts);
+    const agentForTarget = (target) => typeof target === 'string' && target.startsWith('https:') ? httpsAgent : httpAgent;
+    const proxy = httpProxy.createProxyServer({ xfwd: true, secure: false, proxyTimeout: 120_000, agent: httpAgent });
+    // Surface WHICH pod served the request (round-robin made visible).
+    proxy.on('proxyRes', (proxyRes, req) => {
+        const pod = req.__lensmcpPod;
+        if (pod)
+            proxyRes.headers['x-lensmcp-pod'] = pod;
+        // The gateway owns the edge response: drop the upstream framework leak
+        // (e.g. NestJS/Express `x-powered-by`) and brand the hop as the LensMCP gateway.
+        delete proxyRes.headers['x-powered-by'];
+        proxyRes.headers['server'] = 'LensMCP';
+    });
+    // Aggregated host→service traffic edges, flushed to the lens periodically —
+    // the request-derived topology (nodes/edges appear from observed traffic).
+    const edges = new Map();
+    const recordEdge = (route, ms, status, caller) => {
+        const host = route.host ?? '(default)';
+        const key = (caller ? caller + '⇒' : '') + host + '→' + route.project;
+        const e = edges.get(key) ?? { host, project: route.project, caller, count: 0, totalMs: 0, errors: 0 };
+        e.count += 1;
+        e.totalMs += ms;
+        if (status >= 500 || status === 0)
+            e.errors += 1;
+        edges.set(key, e);
+    };
+    const edgeFlusher = setInterval(() => {
+        for (const e of edges.values()) {
+            const avgMs = Math.round(e.totalMs / e.count);
+            emit(e.errors ? 'warning' : 'info', `traffic ${e.host} → ${e.project}: ${e.count} req, ~${avgMs}ms${e.errors ? `, ${e.errors} errors` : ''}`, `gateway-traffic:${e.host}->${e.project}`, { kind: 'gateway-traffic', host: e.host, project: e.project, caller: e.caller, count: e.count, avgMs, errors: e.errors });
+        }
+        edges.clear();
+    }, options.trafficFlushMs ?? 5000);
+    edgeFlusher.unref();
+    const forward = async (route, req, res) => {
+        const startedAt = Date.now();
+        const trace = req.__lensmcpTrace;
+        res.on('close', () => {
+            recordEdge(route, Date.now() - startedAt, res.statusCode ?? 0, req.__lensmcpCaller);
+            finishTrace(trace, route.project, res.statusCode ?? 0, {
+                ...(req.__lensmcpPod ? { pod: req.__lensmcpPod } : {}),
+                ...(req.__lensmcpCaller ? { caller: req.__lensmcpCaller } : {}),
+            });
+        });
+        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix)) {
+            req.url = route.prependPrefix + (req.url ?? '/');
+        }
+        if (route.pool) {
+            const svc = route.svc;
+            if (svc) {
+                svc.lastUsed = Date.now();
+                svc.inflight += 1;
+                res.on('close', () => { svc.inflight -= 1; svc.lastUsed = Date.now(); });
+            }
+            let sock = pickSock(route.pool, scanTtlMs);
+            if (!sock && svc) {
+                // scale from zero: start the service, buffer this request until pods are up
+                traceStep(trace, 'cold-start', { project: route.project });
+                if (await ensureUp(svc))
+                    sock = pickSock(route.pool, scanTtlMs);
+            }
+            if (svc)
+                scaleUp(svc);
+            if (!sock) {
+                res.writeHead(503, { 'content-type': 'text/plain; charset=utf-8' });
+                const errs = svc?.lastErrors?.length ? '\n\nService errors (from build/runtime):\n  ' + svc.lastErrors.join('\n  ') : '';
+                res.end(`Service ${route.project} is not reachable (no pods in ${route.pool.dir}).${errs}\nFix the error — the watcher recovers and the next request retries automatically.`);
+                return;
+            }
+            req.__lensmcpPod = path.basename(sock, '.sock');
+            traceStep(trace, 'pod-select', { pod: path.basename(sock, '.sock'), pods: route.pool.socks.length, strategy: 'round-robin' });
+            proxy.web(req, res, { target: { socketPath: sock }, autoRewrite: true, agent: httpAgent }, (err) => {
+                route.pool.socks = route.pool.socks.filter((s) => s !== sock); // drop dead pod until rescan
+                route.pool.idx = -1; // removing an element shifts indices — reset so the next pick walks cleanly from 0
+                try {
+                    fs.unlinkSync(sock);
+                }
+                catch { /* already gone */ } // a refused sock is a GHOST file — remove it so rescans stay truthful
+                console.error(`[gateway] pod ${path.basename(sock)} of ${route.project}:`, err.message);
+                res.statusCode = 502;
+                res.end(`Pod ${path.basename(sock)} of ${route.project} unavailable — retry.`);
+            });
+            return;
+        }
+        proxy.web(req, res, { target: route.target, autoRewrite: true, changeOrigin: true, agent: agentForTarget(route.target) }, (err) => {
+            console.error(`[gateway] upstream ${route.project} (${route.target}):`, err.message);
+            res.statusCode = 502;
+            res.end(`Upstream ${route.project} unavailable.`);
+        });
+    };
+    // The gateway OWNS these headers — strip any client-supplied copy at ingress so identity
+    // + trust can only originate from the gateway (mirrors the prod gateway's stripTrust).
+    const GATEWAY_OWNED_HEADERS = ['x-internal-token', 'x-api-key-id', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions'];
+    const cors = (req, res) => {
+        const origin = String(req.headers.origin ?? '*');
+        res.setHeader('access-control-allow-origin', origin);
+        res.setHeader('access-control-allow-credentials', 'true');
+        res.setHeader('vary', 'Origin');
+    };
+    const handler = (req, res) => {
+        // STRIP client-supplied trust headers before anything else: the gateway is
+        // the ONLY party allowed to assert `x-internal-token` (came through me) and
+        // `x-api-key-id` (validated caller identity). A client that sends either
+        // must not have it forwarded — the gateway re-stamps them below where
+        // earned. Without this, a keyless/TCP route would leak a spoofed token and
+        // any public route would forward a spoofed caller id to downstream audit.
+        for (const h of GATEWAY_OWNED_HEADERS)
+            delete req.headers[h];
+        const trace = beginTrace(req);
+        if (trace) {
+            traceStep(trace, 'received', { host: trace.host, method: trace.method, url: trace.url });
+            req.__lensmcpTrace = trace;
+        }
+        const route = match(req);
+        if (!route) {
+            // gateway-owned endpoints are cross-origin by design (the app calls
+            // api.<domain>/health from app.<domain>) — answer CORS here; routed
+            // paths inherit CORS from the services themselves.
+            cors(req, res);
+            if (req.method === 'OPTIONS') {
+                res.writeHead(204, {
+                    'access-control-allow-methods': 'GET,HEAD,OPTIONS',
+                    'access-control-allow-headers': String(req.headers['access-control-request-headers'] ?? '*'),
+                });
+                res.end();
+                return;
+            }
+            // Host-root infra: when a host only has PATH-mounted services (pure
+            // microservices under api.<domain>/{prefix}), the gateway itself owns
+            // the root — health aggregation + service discovery, like a real edge.
+            const host = (req.headers.host || '').split(':')[0];
+            const mounted = routes.filter((r) => r.host && (0, manifest_1.hostMatches)(r.host, host) && r.prefix);
+            const url = (req.url ?? '/').split('?')[0];
+            if (mounted.length > 0 && (url === '/health' || url === '/healthz')) {
+                const services = mounted.map((r) => ({
+                    project: r.project,
+                    prefix: r.prefix,
+                    pods: r.pool ? (pickSock(r.pool, 0), r.pool.socks.length) : undefined,
+                    state: r.svc?.state ?? (r.target ? 'up' : 'unknown'),
+                }));
+                const ok = services.every((s) => s.state !== 'down' || true);
+                res.writeHead(ok ? 200 : 503, { 'content-type': 'application/json' });
+                res.end(JSON.stringify({ status: 'ok', host, services }, null, 2));
+                return;
+            }
+            if (mounted.length > 0 && url === '/.well-known/services') {
+                res.writeHead(200, { 'content-type': 'application/json' });
+                res.end(JSON.stringify({ host, services: mounted.map((r) => ({ prefix: r.prefix, project: r.project, internal: !!r.internal })) }, null, 2));
+                return;
+            }
+            res.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' });
+            res.end(mounted.length > 0
+                ? `No service mounted at ${url}. Mounted on ${host}: ${[...new Set(mounted.map((r) => r.prefix))].join(', ')} (+ /health, /.well-known/services)`
+                : 'No gateway route for this hostname.');
+            return;
+        }
+        traceStep(trace, 'route-match', {
+            project: route.project,
+            ...(route.prefix ? { prefix: route.prefix } : {}),
+            ...(route.host ? { routeHost: route.host } : {}),
+            ...(route.internal ? { internal: true } : {}),
+        });
+        if (route.internal) {
+            // service-to-service: validate the CALLER's api key at the boundary
+            const presented = String(req.headers['x-api-key'] ?? '');
+            const caller = keyToProject.get(presented);
+            if (!caller) {
+                traceStep(trace, 'auth', { mode: 'x-api-key', ok: false });
+                finishTrace(trace, route.project, 401);
+                res.writeHead(401, { 'content-type': 'text/plain; charset=utf-8' });
+                res.end('Invalid or missing x-api-key for internal route.');
+                return;
+            }
+            req.headers['x-api-key-id'] = caller; // audit + lens attribution
+            req.__lensmcpCaller = caller;
+            delete req.headers['x-api-key'];
+            traceStep(trace, 'auth', { mode: 'x-api-key', ok: true, caller });
+        }
+        else {
+            // Edge auth (verify the end-user JWT + inject trusted identity + enforce per-route rules),
+            // mirroring the prod gateway. Only enforced when the service declares `cluster.auth`
+            // (opt-in migration — un-migrated services stay open, as before).
+            const policy = route.auth;
+            if (policy) {
+                const url0 = (req.url ?? '/').split('?')[0];
+                const method = (req.method ?? 'GET').toUpperCase();
+                const resolved = (0, manifest_1.authRuleFor)(policy, url0, method);
+                if (resolved.mode === 'jwt') {
+                    const claims = verifyDevJwt(req);
+                    if (!claims) {
+                        traceStep(trace, 'auth', { mode: 'jwt', ok: false });
+                        finishTrace(trace, route.project, 401);
+                        res.writeHead(401, { 'content-type': 'text/plain; charset=utf-8' });
+                        res.end('Unauthorized: invalid or missing token');
+                        return;
+                    }
+                    if (resolved.rule && !(0, manifest_1.evalRule)(resolved.rule, devAttrs(req, claims))) {
+                        traceStep(trace, 'authz', { ok: false });
+                        finishTrace(trace, route.project, 403);
+                        res.writeHead(403, { 'content-type': 'text/plain; charset=utf-8' });
+                        res.end('Forbidden: insufficient permission');
+                        return;
+                    }
+                    stampIdentity(req, claims);
+                    traceStep(trace, 'auth', { mode: 'jwt', ok: true });
+                }
+                else {
+                    traceStep(trace, 'auth', { mode: resolved.mode });
+                }
+            }
+            else {
+                traceStep(trace, 'auth', { mode: 'public', unconfigured: true });
+            }
+            // Custom user middleware (if any) runs AFTER core auth.
+            if (middleware) {
+                try {
+                    middleware(req);
+                }
+                catch (e) {
+                    traceStep(trace, 'auth', { mode: 'middleware', ok: false, reason: e.message });
+                    finishTrace(trace, route.project, 401);
+                    res.writeHead(401, { 'content-type': 'text/plain; charset=utf-8' });
+                    res.end(`Gateway middleware rejected the request: ${e.message}`);
+                    return;
+                }
+            }
+        }
+        if (route.pool && serviceKeys[route.project]) {
+            req.headers['x-internal-token'] = serviceKeys[route.project]; // pods trust the gateway
+        }
+        void forward(route, req, res);
+    };
+    const upgrade = (req, socket, head) => {
+        // WS upgrades enforce the SAME trust model as HTTP — otherwise an
+        // internal.<host> socket would be reachable with no x-api-key, and a
+        // public socket would reach pods without the x-internal-token they expect.
+        for (const h of GATEWAY_OWNED_HEADERS)
+            delete req.headers[h];
+        const route = match(req);
+        if (!route) {
+            socket.destroy();
+            return;
+        }
+        if (route.internal) {
+            const caller = keyToProject.get(String(req.headers['x-api-key'] ?? ''));
+            if (!caller) {
+                socket.destroy();
+                return;
+            } // no valid key → no internal WS
+            req.headers['x-api-key-id'] = caller;
+            delete req.headers['x-api-key'];
+        }
+        else {
+            const policy = route.auth;
+            if (policy) {
+                const resolved = (0, manifest_1.authRuleFor)(policy, (req.url ?? '/').split('?')[0], (req.method ?? 'GET').toUpperCase());
+                if (resolved.mode === 'jwt') {
+                    const claims = verifyDevJwt(req);
+                    if (!claims) {
+                        socket.destroy();
+                        return;
+                    }
+                    if (resolved.rule && !(0, manifest_1.evalRule)(resolved.rule, devAttrs(req, claims))) {
+                        socket.destroy();
+                        return;
+                    }
+                    stampIdentity(req, claims);
+                }
+            }
+            if (middleware) {
+                try {
+                    middleware(req);
+                }
+                catch {
+                    socket.destroy();
+                    return;
+                }
+            }
+        }
+        if (route.pool && serviceKeys[route.project]) {
+            req.headers['x-internal-token'] = serviceKeys[route.project];
+        }
+        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix)) {
+            req.url = route.prependPrefix + (req.url ?? '/');
+        }
+        const target = route.pool
+            ? (() => { const s = pickSock(route.pool, scanTtlMs); return s ? { socketPath: s } : undefined; })()
+            : route.target;
+        if (!target) {
+            socket.destroy();
+            return;
+        }
+        proxy.ws(req, socket, head, { target, changeOrigin: !route.pool, agent: agentForTarget(target) }, () => socket.destroy());
+    };
+    // --- TLS + listeners ------------------------------------------------------------
+    let pem;
+    let caPath;
+    if (https) {
+        // eslint-disable-next-line @typescript-eslint/no-require-imports
+        const basicSsl = require('../../basic-ssl');
+        const cacheDir = path.join(context.root, 'node_modules', '.cache', 'davnx-webpack');
+        const hosts = routes.map((r) => r.host).filter((h) => !!h);
+        pem = basicSsl.getCertificateSync(cacheDir, 'gateway', hosts);
+        caPath = basicSsl.caCertPath(cacheDir);
+    }
+    const servers = [];
+    const boundPorts = [];
+    for (const port of ports) {
+        const server = pem
+            ? // eslint-disable-next-line @typescript-eslint/no-require-imports
+                require('node:https').createServer({ key: pem, cert: pem }, handler)
+            : http.createServer(handler);
+        server.on('upgrade', upgrade);
+        server.on('error', (err) => console.error(`[gateway] port ${port}: ${err.code}`));
+        await new Promise((resolve) => {
+            server.listen(port, () => {
+                const actual = server.address()?.port ?? port;
+                boundPorts.push(actual);
+                console.log(`[gateway] Listening on ${pem ? 'https' : 'http'}://localhost:${actual}`);
+                resolve();
+            });
+        });
+        servers.push(server);
+    }
+    console.log('[gateway] Routes:');
+    for (const r of routes) {
+        const dest = r.pool ? `pods@${r.pool.dir}` : r.target;
+        console.log(`[gateway]   ${r.host ?? '(default)'} → ${dest}${r.prependPrefix ? `  (+${r.prependPrefix})` : ''}${r.internal ? '  [internal: no middleware]' : ''}  [${r.project}]`);
+    }
+    if (caPath)
+        console.log(`[gateway] CA: ${caPath} — one-time setup via the trust executor.`);
+    emit('info', `gateway up: ${routes.length} routes on ${boundPorts.join('/')}`, 'cluster-gateway', {
+        kind: 'gateway-up',
+        ports: boundPorts,
+        routes: routes.map((r) => ({ host: r.host ?? '(default)', project: r.project, mode: r.pool ? 'pods' : 'tcp', internal: !!r.internal, prependPrefix: r.prependPrefix })),
+    });
+    const stop = async () => {
+        if (stopped)
+            return;
+        stopped = true;
+        clearInterval(edgeFlusher);
+        clearInterval(sweeper);
+        emit('info', 'gateway down', 'cluster-gateway', { kind: 'gateway-down' });
+        for (const svc of services)
+            killSpawned(svc, 'gateway shutdown');
+        await Promise.all(servers.map((s) => new Promise((r) => s.close(() => r()))));
+        try {
+            proxy.close?.();
+        }
+        catch { /* ignore */ }
+        httpAgent.destroy();
+        httpsAgent.destroy(); // release pooled upstream sockets
+    };
+    return { ports: boundPorts, routes, services, stop };
+}