@lensmcp/cluster 1.0.0 → 1.2.0

package/executors/gateway/prod-gateway.lib.js

@@ -35,14 +35,39 @@ const reply_from_1 = tslib_1.__importDefault(require("@fastify/reply-from"));
 const manifest_1 = require("./manifest");
 const eid = () => Date.now().toString(36) + (0, node_crypto_1.randomBytes)(6).toString('hex');
 const hdr = (v) => typeof v === 'string' && v !== '' ? v : Array.isArray(v) ? v[0] : undefined;
+// Path-canonicalization guard (CWE-288): a reverse proxy and its backend must agree
+// on the request path, or auth gets evaluated against a DIFFERENT path than the one
+// served (AWS API Gateway trailing-slash bypass, IBM API Connect CVE-2025-13915,
+// Kong/Envoy/NGINX normalization mismatch). We REJECT structurally-encoded paths
+// (encoded dot/slash/backslash/null) outright, and return the DECODED path for auth
+// matching so `/%61dmin` is matched as `/admin` — never skipped because it's encoded.
+const ENCODED_STRUCTURE = /%2e|%2f|%5c|%00|\\/i; // encoded . / \ NUL, or a raw backslash
+/** Canonical path for auth matching, or undefined if the path is unsafe (→ 400 / drop). */
+const canonicalPath = (rawUrl) => {
+    const p = rawUrl.split('?')[0] ?? '/';
+    if (ENCODED_STRUCTURE.test(p))
+        return undefined;
+    let decoded;
+    try {
+        decoded = decodeURIComponent(p);
+    }
+    catch {
+        return undefined;
+    } // malformed %xx
+    if (decoded.includes('\0') || /(^|\/)\.\.(?=\/|$)/.test(decoded))
+        return undefined; // NUL or `..` traversal
+    // Strip fragment (#…) and per-segment matrix params (;…) so we match the path the
+    // BACKEND resolves to — defeats /admin;x=1 and /admin#.css permission-skip bypass (CWE-288).
+    return decoded.split('#')[0].split('/').map((s) => s.split(';')[0]).join('/') || '/';
+};
 async function startProdGateway(opts) {
     const ports = opts.ports?.length ? opts.ports : [opts.tls ? 8443 : 8080];
-    let warnedNoAuth = false;
+    // Default authenticator FAILS CLOSED: a jwt route with no authenticator wired is
+    // rejected, never passed through. Provide opts.authenticate (or a verifier in the
+    // entry) to actually validate tokens.
     const authenticate = opts.authenticate ?? ((mode) => {
-        if (mode === 'jwt' && !warnedNoAuth) {
-            warnedNoAuth = true;
-            console.warn('[gateway] no JWT authenticator configured — jwt routes pass through. Wire opts.authenticate for production.');
-        }
+        if (mode === 'jwt')
+            throw new Error('jwt route but no authenticator configured');
     });
     const emit = (severity, title, fingerprint, raw, ctx) => {
         if (!opts.emit)
@@ -64,11 +89,43 @@ async function startProdGateway(opts) {
         manifests = next;
         routes = (0, manifest_1.buildRouteTable)(next);
         manifestByService = new Map(next.map((m) => [m.service, m]));
+        if (opts.healthChecker) { // monitor every reachable upstream: direct manifest URLs + pooled endpoints
+            const urls = new Set();
+            for (const r of routes)
+                if (r.upstream)
+                    urls.add(r.upstream);
+            for (const u of opts.pods?.endpoints?.() ?? [])
+                urls.add(u);
+            opts.healthChecker.track([...urls]);
+        }
         emit('info', `gateway routes: ${routes.length} on ${manifests.length} services`, 'gateway-up', { kind: 'gateway-up', routes: routes.map((r) => ({ host: r.host ?? '(default)', service: r.service, internal: !!r.internal })) });
     };
     const unwatch = opts.manifests.watch?.((next) => rebuild(next));
     const keyToProject = new Map(Object.entries(opts.serviceKeys ?? {}).map(([p, k]) => [k, p]));
     const versionHeader = (opts.versionHeader ?? 'x-lensmcp-version').toLowerCase();
+    const accessLog = !!opts.accessLog;
+    const clientIpHeader = opts.clientIpHeader?.toLowerCase();
+    // `clientIpHeader` is only trusted from a known front proxy. Default: loopback
+    // only (a co-located cloudflared/sidecar) — an off-proxy attacker hitting the
+    // origin directly cannot spoof the `ip` ABAC attribute.
+    const trustedProxies = opts.clientIpTrustedProxies ?? [];
+    const peerTrusted = (peer) => {
+        if (!peer)
+            return false;
+        const ip = peer.startsWith('::ffff:') ? peer.slice(7) : peer;
+        if (trustedProxies.length === 0)
+            return ip === '127.0.0.1' || peer === '::1';
+        return trustedProxies.some((c) => (0, manifest_1.matchCidr)(ip, c));
+    };
+    /** Resolve the client IP: the trusted front-proxy header if the peer is trusted, else the fallback. */
+    const clientIpOf = (req, fallback) => {
+        if (clientIpHeader && peerTrusted(req.socket?.remoteAddress ?? undefined)) {
+            const h = hdr(req.headers[clientIpHeader]);
+            if (h)
+                return h;
+        }
+        return fallback;
+    };
     // ---- targeted rollout: attributes + sticky device key (cookie → uid header) ----
     const attrHeaders = (opts.attributeHeaders ?? []).map((h) => h.toLowerCase());
     const uidHeader = opts.uidHeader?.toLowerCase();
@@ -124,11 +181,11 @@ async function startProdGateway(opts) {
             return { key: 'd_' + (0, node_crypto_1.randomBytes)(12).toString('base64url'), mint: true };
         return { mint: false };
     };
-    /** Assemble the flat attribute bag a rule evaluates against. */
-    const buildAttrs = (request, principal, key) => {
-        const req = request.raw;
+    /** Assemble the flat attribute bag a rule evaluates against. Shared by the HTTP
+     *  and WebSocket paths so ABAC sees an identical bag on both (no divergence). */
+    const buildAttrs = (req, fallbackIp, principal, key) => {
         const a = { ...(principal ?? {}) };
-        a['ip'] = request.ip;
+        a['ip'] = clientIpOf(req, fallbackIp);
         a['method'] = (req.method ?? 'GET').toUpperCase();
         a['path'] = (req.url ?? '/').split('?')[0];
         if (key)
@@ -152,13 +209,71 @@ async function startProdGateway(opts) {
             e.errors += 1;
         edges.set(key, e);
     };
+    // ---- per-service health rollup (feeds /statusz + the Redis status fan-out) ----
+    // A ring of the last N flush windows, summed at read time → a rolling view that
+    // doesn't flap to "idle" the instant traffic pauses. Derived purely from the
+    // edges the gateway already records, so it's effectively free.
+    const statusWindowCount = Math.max(1, opts.statusWindows ?? 12);
+    const statusWindows = [];
+    const lastSeenAt = new Map();
+    const routeServices = () => [...new Set(routes.map((r) => r.service))];
+    const pushStatusWindow = () => {
+        const w = new Map();
+        for (const e of edges.values()) {
+            const s = w.get(e.service) ?? { requests: 0, errors: 0, totalMs: 0, versions: new Set() };
+            s.requests += e.count;
+            s.errors += e.errors;
+            s.totalMs += e.totalMs;
+            if (e.version)
+                s.versions.add(e.version);
+            w.set(e.service, s);
+        }
+        const now = Date.now();
+        for (const [svc, s] of w)
+            if (s.requests > 0)
+                lastSeenAt.set(svc, now);
+        statusWindows.push(w);
+        while (statusWindows.length > statusWindowCount)
+            statusWindows.shift();
+    };
+    const buildStatusSnapshot = () => {
+        const agg = new Map();
+        for (const w of statusWindows)
+            for (const [svc, s] of w) {
+                const a = agg.get(svc) ?? { requests: 0, errors: 0, totalMs: 0, versions: new Set() };
+                a.requests += s.requests;
+                a.errors += s.errors;
+                a.totalMs += s.totalMs;
+                for (const v of s.versions)
+                    a.versions.add(v);
+                agg.set(svc, a);
+            }
+        return routeServices().map((svc) => {
+            const a = agg.get(svc) ?? { requests: 0, errors: 0, totalMs: 0, versions: new Set() };
+            const hosts = [...new Set(routes.filter((r) => r.service === svc).map((r) => r.host ?? '(default)'))];
+            const er = a.requests ? a.errors / a.requests : 0;
+            let status = a.requests === 0 ? 'idle' : er >= 0.5 ? 'down' : er >= 0.05 ? 'degraded' : 'healthy';
+            // active liveness: if every known upstream for this service is failing probes, it's down (even with no traffic)
+            if (opts.healthChecker) {
+                const svcUrls = routes.filter((r) => r.service === svc && r.upstream).map((r) => r.upstream);
+                if (svcUrls.length && svcUrls.every((u) => !opts.healthChecker.isHealthy(u)))
+                    status = 'down';
+            }
+            return { service: svc, hosts, status, requests: a.requests, errorRate: +er.toFixed(4), avgMs: a.requests ? Math.round(a.totalMs / a.requests) : 0, versions: [...a.versions], lastSeen: lastSeenAt.get(svc) ?? null };
+        });
+    };
     const edgeFlusher = setInterval(() => {
-        if (!opts.emit) {
-            edges.clear();
-            return;
+        pushStatusWindow(); // always: roll the health window (independent of the lens)
+        if (opts.onStatus) {
+            try {
+                opts.onStatus(buildStatusSnapshot());
+            }
+            catch { /* never let a status sink break traffic */ }
         }
-        for (const e of edges.values()) {
-            emit(e.errors ? 'warning' : 'info', `traffic ${e.host} → ${e.service}${e.version ? '@' + e.version : ''}: ${e.count} req`, `gateway-traffic:${e.host}->${e.service}${e.version ? '@' + e.version : ''}`, { kind: 'gateway-traffic', host: e.host, project: e.service, caller: e.caller, version: e.version, count: e.count, avgMs: Math.round(e.totalMs / e.count), errors: e.errors });
+        if (opts.emit) {
+            for (const e of edges.values()) {
+                emit(e.errors ? 'warning' : 'info', `traffic ${e.host} → ${e.service}${e.version ? '@' + e.version : ''}: ${e.count} req`, `gateway-traffic:${e.host}->${e.service}${e.version ? '@' + e.version : ''}`, { kind: 'gateway-traffic', host: e.host, project: e.service, caller: e.caller, version: e.version, count: e.count, avgMs: Math.round(e.totalMs / e.count), errors: e.errors });
+            }
         }
         edges.clear();
     }, opts.trafficFlushMs ?? 5000);
@@ -190,11 +305,18 @@ async function startProdGateway(opts) {
     // The gateway OWNS these headers — drop any client-supplied copy at ingress so identity
     // can only originate from a verified token (closes header spoofing), then re-stamp on forward.
     const TRUST_HEADERS = ['x-internal-token', 'x-api-key-id', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions'];
+    // Also drop client-supplied hop-by-hop control headers: a `Connection: x-internal-token`
+    // would otherwise make undici strip the gateway's OWN injected trust headers before the
+    // upstream (CVE-2026-33805). The proxy manages its own upstream connection semantics.
+    const HOP_BY_HOP = ['connection', 'proxy-connection', 'keep-alive'];
     const stripTrust = (req) => { for (const h of TRUST_HEADERS)
+        delete req.headers[h]; for (const h of HOP_BY_HOP)
         delete req.headers[h]; };
     /** Stamp the verified identity onto the forwarded request — tenant from the `tid` CLAIM, never a client header. */
     const stampIdentity = (req, claims) => {
-        const set = (k, v) => { if (v !== undefined && v !== null && v !== '')
+        // Only stamp SCALAR claims — an object/array sub/tid must never coerce to
+        // "[object Object]" / a CSV tenant header (claim type-confusion).
+        const set = (k, v) => { if ((typeof v === 'string' && v !== '') || typeof v === 'number')
             req.headers[k] = String(v); };
         set('x-user-id', claims['sub']);
         set('x-tenant-id', claims['tid']);
@@ -209,7 +331,10 @@ async function startProdGateway(opts) {
     const resolveUpstream = async (route, trace, pinned) => {
         if (route.upstream)
             return { url: route.upstream };
-        const opt = pinned ? { version: pinned } : undefined;
+        const opt = {
+            ...(pinned ? { version: pinned } : {}),
+            ...(opts.healthChecker ? { isHealthy: (u) => opts.healthChecker.isHealthy(u) } : {}),
+        };
         let up = opts.pods?.pick(route.service, opt);
         if (!up && opts.pods?.ensureUp) {
             step(trace, 'ensure-up', { project: route.service });
@@ -219,10 +344,13 @@ async function startProdGateway(opts) {
         return up;
     };
     const reqState = new WeakMap();
+    const spanByReq = new WeakMap(); // OTel span per in-flight request (ended in onResponse)
     // ---- no-route fallbacks: CORS preflight, health, discovery, 404 ----
     const setCors = (reply, req) => {
+        // These are UNAUTHENTICATED gateway responses (404 / health / route discovery /
+        // preflight). Reflect the Origin for convenience but DO NOT allow credentials —
+        // a reflected origin + credentials would let any site read these cross-origin.
         reply.header('access-control-allow-origin', String(req.headers.origin ?? '*'));
-        reply.header('access-control-allow-credentials', 'true');
         reply.header('vary', 'Origin');
     };
     const handleNoRoute = (request, reply, host, url) => {
@@ -254,12 +382,51 @@ async function startProdGateway(opts) {
         const url = req.url ?? '/';
         const trace = beginTrace(req);
         step(trace, 'received', { host, method: req.method, url });
+        // Reject HTTP/0.9 — a header-less simple request has no Host, defeating host-keyed routing/policy.
+        if (req.httpVersionMajor < 1) {
+            finish(trace, 'gateway', 400);
+            reply.code(400).type('text/plain').send('Bad Request');
+            return reply;
+        }
+        // Reject path-canonicalization attacks BEFORE routing/auth (proxy↔backend mismatch).
+        const cpath = canonicalPath(url);
+        if (cpath === undefined) {
+            step(trace, 'bad-path', {});
+            finish(trace, 'gateway', 400);
+            reply.code(400).type('text/plain').send('Bad Request');
+            return reply;
+        }
+        // Rate limit (opt-in) BEFORE routing/auth, per real client IP — defense-in-depth.
+        if (opts.rateLimiter) {
+            // Key on the trusted client IP (header only from a trusted peer) else the SOCKET ip —
+            // never request.ip, which is the spoofable X-Forwarded-For chain under trustProxy.
+            const d = await opts.rateLimiter.check(clientIpOf(req, req.socket?.remoteAddress ?? request.ip));
+            if (!d.allowed) {
+                reply.header('retry-after', String(Math.ceil(d.retryAfterMs / 1000)));
+                step(trace, 'rate-limited', {});
+                finish(trace, 'gateway', 429);
+                reply.code(429).type('text/plain').send('Too Many Requests');
+                return reply;
+            }
+        }
         const route = (0, manifest_1.matchRoute)(routes, host, url);
         if (!route) {
             handleNoRoute(request, reply, host, url);
             return reply;
         }
         step(trace, 'route-match', { project: route.service, ...(route.prefix ? { prefix: route.prefix } : {}), ...(route.internal ? { internal: true } : {}) });
+        // OTel SERVER span (opt-in): continues the incoming traceparent. Stored now so
+        // onResponse ends it on ANY exit path (auth reject / 503 / forward). Context is
+        // injected into req.headers just before forwarding (below) so downstream parents to it.
+        const span = opts.tracing?.startSpan(req, `gateway ${route.service}`, {
+            'http.request.method': (req.method ?? 'GET'),
+            'url.path': url.split('?')[0],
+            'server.address': host,
+            'lensmcp.service': route.service,
+            ...(route.internal ? { 'lensmcp.internal': true } : {}),
+        });
+        if (span)
+            spanByReq.set(request, span);
         // Verify the end-user JWT ONCE (when a verifier is configured) — reused for both edge
         // authorization and rollout targeting. Internal east-west routes use x-api-key instead.
         const claims = route.internal ? undefined : opts.identify?.(req);
@@ -278,7 +445,7 @@ async function startProdGateway(opts) {
         }
         else {
             const m = manifestByService.get(route.service);
-            const resolved = m ? (0, manifest_1.authRuleForPath)(m, url, req.method ?? 'GET', false) : { mode: route.auth };
+            const resolved = m ? (0, manifest_1.authRuleForPath)(m, cpath, req.method ?? 'GET', false) : { mode: route.auth };
             if (resolved.mode === 'jwt') {
                 if (opts.identify) {
                     if (!claims) {
@@ -300,7 +467,8 @@ async function startProdGateway(opts) {
                     }
                 }
                 // edge ABAC: the required permission/rule must hold against the VERIFIED claims.
-                if (resolved.rule && claims && !(0, manifest_1.evalRule)(resolved.rule, buildAttrs(request, claims, undefined))) {
+                // A present rule MUST be satisfied by VERIFIED claims. No claims (no verifier) → cannot satisfy → DENY (fail closed).
+                if (resolved.rule && (!claims || !(0, manifest_1.evalRule)(resolved.rule, buildAttrs(req, request.ip, claims, undefined)))) {
                     step(trace, 'authz', { ok: false });
                     finish(trace, route.service, 403);
                     reply.code(403).type('text/plain').send('Forbidden: insufficient permission');
@@ -316,8 +484,12 @@ async function startProdGateway(opts) {
         }
         if (opts.serviceKeys?.[route.service])
             req.headers['x-internal-token'] = opts.serviceKeys[route.service];
-        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix))
-            req.url = route.prependPrefix + (req.url ?? '/');
+        if (route.prependPrefix) {
+            const u = req.url ?? '/';
+            const pp = route.prependPrefix;
+            if (u !== pp && !u.startsWith(pp + '/'))
+                req.url = pp + u;
+        } // segment-boundary, not bare startsWith (/apidoc must not escape /api)
         // ── version selection: explicit pin → ABAC rule → sticky-key weighted → SWRR ──
         // (targeting/stickiness only for external routes; internal east-west has no "device")
         const explicit = hdr(req.headers[versionHeader]);
@@ -327,7 +499,7 @@ async function startProdGateway(opts) {
             const sk = resolveStickyKey(req);
             if (sk.mint)
                 mintKey = sk.key;
-            const attrs = buildAttrs(request, claims, sk.key);
+            const attrs = buildAttrs(req, request.ip, claims, sk.key);
             const targeted = opts.pods?.resolveVersion?.(route.service, attrs);
             if (targeted) {
                 pinned = targeted.version;
@@ -356,6 +528,9 @@ async function startProdGateway(opts) {
         const upLabel = up.url;
         reqState.set(request, { route, caller, startedAt: Date.now(), trace, version: up.version, upLabel });
         step(trace, up.version ? 'rollout' : 'upstream', { upstream: upLabel, ...(up.version ? { version: up.version, ...(explicit ? { pinned: true } : {}) } : {}) });
+        // Propagate the gateway span's context downstream (W3C traceparent on the forwarded request).
+        if (span)
+            opts.tracing.inject(span, req.headers);
         // No source → reply-from forwards the (prefix-rewritten) req.url to `origin`.
         return reply.from(undefined, {
             getUpstream: () => origin,
@@ -366,6 +541,9 @@ async function startProdGateway(opts) {
                 // The gateway owns the edge response: drop the upstream framework leak
                 // (e.g. NestJS/Express `x-powered-by`) and brand the hop as the LensMCP gateway.
                 delete headers['x-powered-by'];
+                delete headers['connection'];
+                delete headers['keep-alive'];
+                delete headers['proxy-connection']; // strip hop-by-hop from the upstream response
                 headers['server'] = 'LensMCP';
                 if (mintKey) { // first contact → stamp the device cookie (appended to any upstream Set-Cookie)
                     const ex = headers['set-cookie'];
@@ -376,7 +554,8 @@ async function startProdGateway(opts) {
             },
             onError: (rep, { error }) => {
                 opts.pods?.drop?.(route.service, up);
-                rep.code(502).type('text/plain').send(`Upstream ${route.service} unavailable — ${error.message}`);
+                void error; // don't reflect the raw upstream error (host:port / errno) to the client
+                rep.code(502).type('text/plain').send(`Upstream ${route.service} unavailable.`);
             },
         });
     };
@@ -384,6 +563,11 @@ async function startProdGateway(opts) {
     const upgrade = (req, socket, head) => {
         stripTrust(req);
         const host = (req.headers.host || '').split(':')[0];
+        const cpath = canonicalPath(req.url ?? '/');
+        if (cpath === undefined) {
+            socket.destroy();
+            return;
+        } // reject path-canonicalization attacks
         const route = (0, manifest_1.matchRoute)(routes, host, req.url ?? '/');
         if (!route) {
             socket.destroy();
@@ -400,7 +584,7 @@ async function startProdGateway(opts) {
         }
         else {
             const m = manifestByService.get(route.service);
-            const resolved = m ? (0, manifest_1.authRuleForPath)(m, req.url ?? '/', req.method ?? 'GET', false) : { mode: route.auth };
+            const resolved = m ? (0, manifest_1.authRuleForPath)(m, cpath, req.method ?? 'GET', false) : { mode: route.auth };
             if (resolved.mode === 'jwt') {
                 if (opts.identify) {
                     if (!claims) {
@@ -417,14 +601,19 @@ async function startProdGateway(opts) {
                         return;
                     }
                 }
-                if (resolved.rule && claims && !(0, manifest_1.evalRule)(resolved.rule, { ...claims, ip: req.socket.remoteAddress ?? '', method: req.method ?? 'GET', path: (req.url ?? '/').split('?')[0] })) {
+                // Same fail-closed ABAC + identical attribute bag as the HTTP path (no divergence).
+                if (resolved.rule && (!claims || !(0, manifest_1.evalRule)(resolved.rule, buildAttrs(req, req.socket.remoteAddress ?? '', claims, undefined)))) {
                     socket.destroy();
                     return;
                 }
             }
         }
-        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix))
-            req.url = route.prependPrefix + (req.url ?? '/');
+        if (route.prependPrefix) {
+            const u = req.url ?? '/';
+            const pp = route.prependPrefix;
+            if (u !== pp && !u.startsWith(pp + '/'))
+                req.url = pp + u;
+        } // segment-boundary, not bare startsWith (/apidoc must not escape /api)
         const pinned = hdr(req.headers[versionHeader]);
         void (async () => {
             const up = await resolveUpstream(route, undefined, pinned);
@@ -436,8 +625,10 @@ async function startProdGateway(opts) {
             const upstream = net.connect(Number(u.port) || (u.protocol === 'https:' ? 443 : 80), u.hostname, () => {
                 // rebuild the upgrade request from rawHeaders (verbatim), minus the trust headers
                 // we own (incl. client identity), plus the validated caller + target token + identity.
-                const strip = new Set(['x-internal-token', 'x-api-key-id', 'x-api-key', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions']);
-                const lines = [`${req.method} ${req.url} HTTP/1.1`];
+                // Strip the trust headers AND the client's hop-by-hop control (connection/keep-alive),
+                // so a `Connection: x-internal-token` can't strip the gateway's injected headers downstream.
+                const strip = new Set(['x-internal-token', 'x-api-key-id', 'x-api-key', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions', 'connection', 'proxy-connection', 'keep-alive']);
+                const lines = [`${req.method} ${req.url} HTTP/1.1`, 'Connection: Upgrade']; // gateway-owned, clean upgrade
                 for (let i = 0; i < req.rawHeaders.length; i += 2) {
                     const k = req.rawHeaders[i];
                     if (!strip.has(k.toLowerCase()))
@@ -470,9 +661,43 @@ async function startProdGateway(opts) {
             socket.on('error', () => upstream.destroy());
         })();
     };
+    // ---- opt-in access log: one Cloud-Logging-shaped JSON line per response ----
+    // Covers EVERY response (incl. 401/404/503 that never reached an upstream),
+    // not just forwarded ones; `reply.elapsedTime` gives latency without our own timer.
+    const logAccess = (request, reply, st) => {
+        const req = request.raw;
+        const status = reply.statusCode ?? 0;
+        const host = (req.headers.host || '').split(':')[0];
+        const path = (req.url || '/').split('?')[0];
+        const ms = Math.round(reply.elapsedTime);
+        try {
+            console.log(JSON.stringify({
+                severity: status >= 500 ? 'ERROR' : status >= 400 ? 'WARNING' : 'INFO',
+                time: new Date().toISOString(),
+                message: `${req.method} ${host}${path} ${status} ${ms}ms`,
+                method: req.method, host, path, status, ms,
+                ...(st ? { service: st.route.service, upstream: st.upLabel } : {}),
+                ...(st?.version ? { version: st.version } : {}),
+                ...(st?.caller ? { caller: st.caller } : {}),
+                ...(req.headers['x-request-id'] ? { requestId: String(req.headers['x-request-id']) } : {}),
+            }));
+        }
+        catch { /* never let logging break the response */ }
+    };
     // ---- build one Fastify app per port (shared route table + providers) ----
     const buildApp = async () => {
-        const app = (0, fastify_1.default)({ logger: false, ...(opts.trustProxyIp ? { trustProxy: true } : {}), ...(opts.tls ? { https: { key: opts.tls.key, cert: opts.tls.cert } } : {}) });
+        const keepAlive = opts.server?.keepAliveTimeout ?? 620_000; // > GCP external ALB 600s backend keep-alive
+        const app = (0, fastify_1.default)({
+            logger: false,
+            keepAliveTimeout: keepAlive,
+            requestTimeout: opts.server?.requestTimeout ?? 0, // 0 = no inbound request timeout (long-lived streaming)
+            ...(opts.server?.connectionTimeout !== undefined ? { connectionTimeout: opts.server.connectionTimeout } : {}),
+            ...(opts.trustProxyIp ? { trustProxy: true } : {}),
+            ...(opts.tls ? { https: { key: opts.tls.key, cert: opts.tls.cert } } : {}),
+        });
+        // headersTimeout must stay above keepAliveTimeout, else Node may close an idle
+        // keep-alive socket mid-request under load. Default: just above keepAlive.
+        app.server.headersTimeout = opts.server?.headersTimeout ?? keepAlive + 10_000;
         // Stream every body straight through — never parse/buffer (this is a proxy).
         app.removeAllContentTypeParsers();
         app.addContentTypeParser('*', (_req, payload, done) => done(null, payload));
@@ -482,7 +707,15 @@ async function startProdGateway(opts) {
                 connections: opts.undici?.connections ?? 256,
                 pipelining: opts.undici?.pipelining ?? 1,
                 keepAliveTimeout: opts.undici?.keepAliveTimeout ?? 60_000,
-                connect: { rejectUnauthorized: false }, // upstreams may use the local CA / self-signed (mirrors http-proxy secure:false)
+                bodyTimeout: opts.undici?.bodyTimeout ?? 0, // 0 = unlimited — never cut long-lived SSE / streaming MCP responses
+                headersTimeout: opts.undici?.headersTimeout ?? 60_000, // upstream must send response headers within 60s
+                ...(opts.undici?.allowH2 ? { allowH2: true } : {}), // multiplex many MCP streams over few h2 conns (Cloud Run speaks h2)
+                connect: {
+                    rejectUnauthorized: opts.undici?.rejectUnauthorized ?? false, // local-CA/self-signed by default; set true for verified east-west on GCP
+                    ...(opts.undici?.cert ? { cert: opts.undici.cert } : {}), // ── mTLS: present a client cert to upstreams ──
+                    ...(opts.undici?.key ? { key: opts.undici.key } : {}),
+                    ...(opts.undici?.ca ? { ca: opts.undici.ca } : {}), // trust a private CA for the server side
+                },
             },
         });
         // Host-agnostic operational probes (answered before routing).
@@ -491,6 +724,38 @@ async function startProdGateway(opts) {
             const ready = routes.length > 0;
             reply.code(ready ? 200 : 503).type('text/plain').send(ready ? 'ready' : 'no routes loaded');
         });
+        // Per-service health rollup (opt-in; token-guarded). For a status page that
+        // prefers HTTP over the Redis `status:*` fan-out. Off by default → 404 (don't
+        // leak topology on a public edge unless explicitly enabled).
+        app.get('/statusz', (request, reply) => {
+            // Fail closed: /statusz is served ONLY when both enabled AND a token is set
+            // (never expose topology/health unauthenticated). Constant-time token compare.
+            if (!opts.statusEndpoint || !opts.statusToken) {
+                reply.code(404).type('text/plain').send('not found');
+                return;
+            }
+            const tok = request.query?.token ?? hdr(request.raw.headers['x-status-token']) ?? '';
+            const a = Buffer.from(tok), b = Buffer.from(opts.statusToken);
+            if (a.length !== b.length || !(0, node_crypto_1.timingSafeEqual)(a, b)) {
+                reply.code(401).type('text/plain').send('unauthorized');
+                return;
+            }
+            reply.type('application/json').send(JSON.stringify({ generatedAt: Date.now(), services: buildStatusSnapshot(), endpoints: opts.healthChecker?.snapshot() ?? [] }));
+        });
+        // Process self-metrics (event-loop lag / rss / cpu / req rates) — same token gate as /statusz.
+        app.get('/metricsz', (request, reply) => {
+            if (!opts.metrics || !opts.statusToken) {
+                reply.code(404).type('text/plain').send('not found');
+                return;
+            }
+            const tok = request.query?.token ?? hdr(request.raw.headers['x-status-token']) ?? '';
+            const a = Buffer.from(tok), b = Buffer.from(opts.statusToken);
+            if (a.length !== b.length || !(0, node_crypto_1.timingSafeEqual)(a, b)) {
+                reply.code(401).type('text/plain').send('unauthorized');
+                return;
+            }
+            reply.type('application/json').send(JSON.stringify(opts.metrics.snapshot()));
+        });
         app.addHook('onRequest', (request, _reply, done) => { stripTrust(request.raw); done(); });
         app.addHook('onResponse', (request, reply, done) => {
             const st = reqState.get(request);
@@ -499,6 +764,14 @@ async function startProdGateway(opts) {
                 finish(st.trace, st.route.service, reply.statusCode ?? 0, { upstream: st.upLabel, ...(st.version ? { version: st.version } : {}), ...(st.caller ? { caller: st.caller } : {}) });
                 reqState.delete(request);
             }
+            const sp = spanByReq.get(request);
+            if (sp) {
+                opts.tracing.end(sp, reply.statusCode ?? 0, { ...(st?.upLabel ? { 'upstream.url': st.upLabel } : {}), ...(st?.version ? { 'lensmcp.version': st.version } : {}) });
+                spanByReq.delete(request);
+            }
+            opts.metrics?.recordRequest(reply.statusCode ?? 0);
+            if (accessLog)
+                logAccess(request, reply, st);
             done();
         });
         app.all('/', handler);
@@ -518,16 +791,21 @@ async function startProdGateway(opts) {
         console.log(`[gateway] listening on ${opts.tls ? 'https' : 'http'}://0.0.0.0:${bound}`);
         apps.push(app);
     }
+    opts.healthChecker?.start(); // begin probing now that the route table (and tracked URLs) is warm
     let stopped = false;
     return {
         ports: boundPorts,
         routes: () => routes,
+        status: buildStatusSnapshot,
         stop: async () => {
             if (stopped)
                 return;
             stopped = true;
             clearInterval(edgeFlusher);
             unwatch?.();
+            opts.healthChecker?.stop();
+            opts.rateLimiter?.stop();
+            opts.metrics?.stop();
             emit('info', 'gateway down', 'gateway-down', { kind: 'gateway-down' });
             await Promise.all(apps.map((a) => a.close())); // reply-from closes its undici pool on close
         },

package/executors/gateway/providers-prod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"providers-prod.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/providers-prod.ts"],"names":[],"mappings":"AAkBA,OAAO,EAEL,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAiB,KAAK,aAAa,EAAE,KAAK,UAAU,EAChH,MAAM,YAAY,CAAC;AAEpB,iFAAiF;AACjF,MAAM,WAAW,WAAW;IAAG,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE;AAC/E,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC,CAAC;AAIzE,4DAA4D;AAC5D,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAEnF;AAgBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAA;CAAO,GAC7D,gBAAgB,GAAG;IAAE,IAAI,EAAE,MAAM,IAAI,CAAA;CAAE,CA+BzC;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,YAAY,GAClB,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,CAAA;CAAE,CAkDxD;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAClC,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAK,IAAI,CAAA;CAAE,CAapE"}
+{"version":3,"file":"providers-prod.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/providers-prod.ts"],"names":[],"mappings":"AAkBA,OAAO,EAEL,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAiB,KAAK,aAAa,EAAE,KAAK,UAAU,EAChH,MAAM,YAAY,CAAC;AAEpB,iFAAiF;AACjF,MAAM,WAAW,WAAW;IAAG,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE;AAC/E,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC,CAAC;AAIzE,4DAA4D;AAC5D,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAEnF;AAgBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAA;CAAO,GAC7D,gBAAgB,GAAG;IAAE,IAAI,EAAE,MAAM,IAAI,CAAA;CAAE,CA+BzC;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,YAAY,GAClB,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,CAAA;CAAE,CAwExD;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAClC,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAK,IAAI,CAAA;CAAE,CAapE"}

package/executors/gateway/providers-prod.js

@@ -139,20 +139,53 @@ function rolloutPodProvider(table) {
             const st = svc.get(service);
             if (!st || st.cohorts.length === 0)
                 return undefined;
-            let ci = -1;
-            if (opts?.version)
-                ci = st.cohorts.findIndex((c) => c.version === opts.version && c.urls.length > 0);
-            if (ci < 0) {
-                ci = st.select();
-                // chosen cohort drained/empty → fall back to any cohort with endpoints
-                if (!st.cohorts[ci] || st.cohorts[ci].urls.length === 0)
-                    ci = firstNonEmpty(st);
-            }
-            const cohort = st.cohorts[ci];
-            if (!cohort || cohort.urls.length === 0)
+            const isHealthy = opts?.isHealthy;
+            // round-robin the chosen cohort, preferring healthy endpoints; undefined if none pass.
+            const pickUrl = (ci, gate) => {
+                const cohort = st.cohorts[ci];
+                if (!cohort || cohort.urls.length === 0)
+                    return undefined;
+                const n = cohort.urls.length;
+                for (let i = 0; i < n; i++) {
+                    st.rr[ci] = (st.rr[ci] + 1) % n;
+                    const u = cohort.urls[st.rr[ci]];
+                    if (!gate || gate(u))
+                        return u;
+                }
                 return undefined;
-            st.rr[ci] = (st.rr[ci] + 1) % cohort.urls.length;
-            return { url: cohort.urls[st.rr[ci]], version: cohort.version };
+            };
+            const pinnedCi = opts?.version ? st.cohorts.findIndex((c) => c.version === opts.version && c.urls.length > 0) : -1;
+            if (pinnedCi >= 0) {
+                const url = pickUrl(pinnedCi, isHealthy) ?? pickUrl(pinnedCi); // fail-open WITHIN the pinned cohort
+                return url !== undefined ? { url, version: st.cohorts[pinnedCi].version } : undefined;
+            }
+            // not pinned (or unknown version) → weighted selection (unchanged contract)
+            let ci = st.select();
+            if (!st.cohorts[ci] || st.cohorts[ci].urls.length === 0)
+                ci = firstNonEmpty(st);
+            let url = pickUrl(ci, isHealthy);
+            // chosen cohort has no healthy endpoint → try any cohort that does
+            if (url === undefined && isHealthy) {
+                for (let i = 0; i < st.cohorts.length; i++) {
+                    const u = pickUrl(i, isHealthy);
+                    if (u !== undefined) {
+                        ci = i;
+                        url = u;
+                        break;
+                    }
+                }
+            }
+            if (url === undefined)
+                url = pickUrl(ci); // fail-open: everything looks down → still route
+            return url !== undefined ? { url, version: st.cohorts[ci].version } : undefined;
+        },
+        endpoints() {
+            const urls = new Set();
+            for (const st of svc.values())
+                for (const c of st.cohorts)
+                    for (const u of c.urls)
+                        urls.add(u);
+            return [...urls];
         },
         // first targeting rule that matches the request's attributes (sticky-by-attribute)
         resolveVersion(service, attrs) {