@lensmcp/cluster 1.0.0

package/executors/gateway/prod-gateway.lib.d.ts

@@ -0,0 +1,58 @@
+import * as http from 'node:http';
+import { type AuthMode, type ManifestProvider, type PodProvider, type Route } from './manifest';
+export interface ProdGatewayOptions {
+    /** Ports to bind. Default `[8080]` (or `[8443]` with TLS). One Fastify
+     *  instance per port; all share the route table + providers. */
+    ports?: number[];
+    manifests: ManifestProvider;
+    /** Resolves a service → upstream when a route has no direct `upstream`. */
+    pods?: PodProvider;
+    /** service → its `x-internal-token` (from a k8s Secret / Vault in prod). */
+    serviceKeys?: Record<string, string>;
+    /** Validate a `jwt`-mode request; THROW to reject (→ 401). Default: a
+     *  pass-through with a one-time warning — wire a real JWKS validator in prod. */
+    authenticate?: (mode: AuthMode, req: http.IncomingMessage) => void;
+    tls?: {
+        key: string | Buffer;
+        cert: string | Buffer;
+    };
+    /** Lens bus sink — OPT-IN. Omit (default) for zero observability overhead. */
+    emit?: (event: Record<string, unknown>) => void;
+    /** Traffic-edge flush period (ms). Default 5000. */
+    trafficFlushMs?: number;
+    /** Request header that PINS a rollout cohort (sticky canary / debugging).
+     *  Default `x-lensmcp-version`. The chosen version is echoed on the response. */
+    versionHeader?: string;
+    /** undici pool tuning passed to reply-from (connections, pipelining…). */
+    undici?: {
+        connections?: number;
+        pipelining?: number;
+        keepAliveTimeout?: number;
+    };
+    /** Return verified subject attributes (e.g. JWT claims) for ABAC routing.
+     *  MUST verify (signature/exp) — forged claims must not reach the rules. */
+    identify?: (req: http.IncomingMessage) => Record<string, unknown> | undefined;
+    /** Allowlist of request headers exposed to rules as `header.<name>` attributes. */
+    attributeHeaders?: string[];
+    /** Take the `ip` attribute from `X-Forwarded-For` (behind a trusted LB) via
+     *  Fastify `trustProxy`; default false → socket IP. */
+    trustProxyIp?: boolean;
+    /** Gateway-issued device cookie for sticky bucketing (anonymous-friendly).
+     *  Omit to disable cookie stickiness. `secret` enables HMAC signing. */
+    cookie?: {
+        name?: string;
+        secret?: string;
+        domain?: string;
+        maxAge?: number;
+    };
+    /** Header carrying a client device/user id — the sticky key when no cookie
+     *  is present (native/mobile/API clients), e.g. `x-device-id`. */
+    uidHeader?: string;
+}
+export interface ProdGatewayHandle {
+    ports: number[];
+    routes: () => Route[];
+    stop: () => Promise<void>;
+}
+export declare function startProdGateway(opts: ProdGatewayOptions): Promise<ProdGatewayHandle>;
+//# sourceMappingURL=prod-gateway.lib.d.ts.map

package/executors/gateway/prod-gateway.lib.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prod-gateway.lib.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/prod-gateway.lib.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EAEL,KAAK,QAAQ,EAAc,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAqB,KAAK,KAAK,EAClG,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,kBAAkB;IACjC;oEACgE;IAChE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,2EAA2E;IAC3E,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC;qFACiF;IACjF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACnE,GAAG,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACtD,8EAA8E;IAC9E,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChD,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;qFACiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,MAAM,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAGlF;gFAC4E;IAC5E,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9E,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;4EACwE;IACxE,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9E;sEACkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,KAAK,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAYD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoY3F"}

package/executors/gateway/prod-gateway.lib.js

@@ -0,0 +1,535 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startProdGateway = startProdGateway;
+const tslib_1 = require("tslib");
+/**
+ * Production gateway runtime — Fastify + @fastify/reply-from (undici) engine.
+ *
+ * The SAME routing brain as dev (shared `matchRoute` / `buildRouteTable` /
+ * manifest schema) composed over the prod providers (a manifest source +
+ * endpoint resolution). The HTTP transport differs from dev on purpose: dev
+ * uses node:http + http-proxy (it manages unix-socket pods + scale-from-zero),
+ * prod uses Fastify + reply-from's undici pool — connection-reusing, HTTP/1.1
+ * pipelined, materially faster than node-http-proxy for the high-RPS edge.
+ *
+ * The exported contract (`startProdGateway` / `ProdGatewayOptions` /
+ * `ProdGatewayHandle`) is unchanged from the previous engine, so the entry,
+ * the e2e suite, and the perf suite drive it identically — the swap is proven
+ * behind a stable seam.
+ *
+ * It keeps the dev trust boundary verbatim — strip client-supplied trust
+ * headers, validate `x-api-key` on internal routes, inject the target's
+ * `x-internal-token` — and the header-gated `gateway-request` flow trace, so
+ * the lens (when ENABLED via opts.emit; off by default) sees identical events.
+ * Per-request upstream selection (rollout cohort + sticky version) rides on
+ * reply-from's `getUpstream` hook. WebSocket upgrades are proxied raw (a
+ * net-level pipe) with the same trust model — no http-proxy in the prod path.
+ *
+ * No autoscale / idle-kill / process management here — prod pods are
+ * always-on (k8s keeps them up); that machinery is the dev PodProvider's.
+ */
+const node_crypto_1 = require("node:crypto");
+const net = tslib_1.__importStar(require("node:net"));
+const fastify_1 = tslib_1.__importDefault(require("fastify"));
+const reply_from_1 = tslib_1.__importDefault(require("@fastify/reply-from"));
+const manifest_1 = require("./manifest");
+const eid = () => Date.now().toString(36) + (0, node_crypto_1.randomBytes)(6).toString('hex');
+const hdr = (v) => typeof v === 'string' && v !== '' ? v : Array.isArray(v) ? v[0] : undefined;
+async function startProdGateway(opts) {
+    const ports = opts.ports?.length ? opts.ports : [opts.tls ? 8443 : 8080];
+    let warnedNoAuth = false;
+    const authenticate = opts.authenticate ?? ((mode) => {
+        if (mode === 'jwt' && !warnedNoAuth) {
+            warnedNoAuth = true;
+            console.warn('[gateway] no JWT authenticator configured — jwt routes pass through. Wire opts.authenticate for production.');
+        }
+    });
+    const emit = (severity, title, fingerprint, raw, ctx) => {
+        if (!opts.emit)
+            return; // lens off by default — no work, no allocation beyond the call
+        try {
+            opts.emit({
+                id: eid(), sessionId: 'pending', timestamp: Date.now(),
+                source: 'gateway', category: 'cluster', severity,
+                context: { sessionId: 'pending', ...ctx }, fingerprint, title, raw,
+            });
+        }
+        catch { /* never let the bus break traffic */ }
+    };
+    // ---- route table (rebuilt atomically from the manifest source) ----
+    let manifests = opts.manifests.list();
+    let routes = (0, manifest_1.buildRouteTable)(manifests);
+    let manifestByService = new Map(manifests.map((m) => [m.service, m]));
+    const rebuild = (next) => {
+        manifests = next;
+        routes = (0, manifest_1.buildRouteTable)(next);
+        manifestByService = new Map(next.map((m) => [m.service, m]));
+        emit('info', `gateway routes: ${routes.length} on ${manifests.length} services`, 'gateway-up', { kind: 'gateway-up', routes: routes.map((r) => ({ host: r.host ?? '(default)', service: r.service, internal: !!r.internal })) });
+    };
+    const unwatch = opts.manifests.watch?.((next) => rebuild(next));
+    const keyToProject = new Map(Object.entries(opts.serviceKeys ?? {}).map(([p, k]) => [k, p]));
+    const versionHeader = (opts.versionHeader ?? 'x-lensmcp-version').toLowerCase();
+    // ---- targeted rollout: attributes + sticky device key (cookie → uid header) ----
+    const attrHeaders = (opts.attributeHeaders ?? []).map((h) => h.toLowerCase());
+    const uidHeader = opts.uidHeader?.toLowerCase();
+    const cookieOn = !!opts.cookie;
+    const cookieName = opts.cookie?.name ?? 'lensmcp_did';
+    const cookieSecret = opts.cookie?.secret;
+    const cookieDomain = opts.cookie?.domain;
+    const cookieMaxAge = opts.cookie?.maxAge ?? 31_536_000; // 1y
+    if (cookieOn && !cookieSecret)
+        console.warn('[gateway] device cookie has no secret — values are UNSIGNED (set cookie.secret to prevent bucket-grinding).');
+    const signId = (id) => (0, node_crypto_1.createHmac)('sha256', cookieSecret).update(id).digest('base64url');
+    const signedCookie = (id) => (cookieSecret ? `${id}.${signId(id)}` : id);
+    const verifyCookie = (val) => {
+        if (!cookieSecret)
+            return val || undefined; // unsigned mode
+        const i = val.lastIndexOf('.');
+        if (i < 0)
+            return undefined;
+        const id = val.slice(0, i);
+        const a = Buffer.from(val.slice(i + 1)), b = Buffer.from(signId(id));
+        return a.length === b.length && (0, node_crypto_1.timingSafeEqual)(a, b) ? id : undefined;
+    };
+    const parseCookie = (header, name) => {
+        if (!header)
+            return undefined;
+        for (const part of header.split(';')) {
+            const eq = part.indexOf('=');
+            if (eq > 0 && part.slice(0, eq).trim() === name)
+                return decodeURIComponent(part.slice(eq + 1).trim());
+        }
+        return undefined;
+    };
+    const mintCookieHeader = (id) => {
+        let c = `${cookieName}=${encodeURIComponent(signedCookie(id))}; Path=/; Max-Age=${cookieMaxAge}; HttpOnly; SameSite=Lax; Secure`;
+        if (cookieDomain)
+            c += `; Domain=${cookieDomain}`;
+        return c;
+    };
+    /** Resolve the sticky key: valid cookie → UID header → freshly minted (cookie clients). */
+    const resolveStickyKey = (req) => {
+        if (cookieOn) {
+            const c = parseCookie(req.headers.cookie, cookieName);
+            const id = c ? verifyCookie(c) : undefined;
+            if (id)
+                return { key: id, mint: false };
+        }
+        if (uidHeader) {
+            const u = hdr(req.headers[uidHeader]);
+            if (u)
+                return { key: u, mint: false };
+        }
+        if (cookieOn)
+            return { key: 'd_' + (0, node_crypto_1.randomBytes)(12).toString('base64url'), mint: true };
+        return { mint: false };
+    };
+    /** Assemble the flat attribute bag a rule evaluates against. */
+    const buildAttrs = (request, principal, key) => {
+        const req = request.raw;
+        const a = { ...(principal ?? {}) };
+        a['ip'] = request.ip;
+        a['method'] = (req.method ?? 'GET').toUpperCase();
+        a['path'] = (req.url ?? '/').split('?')[0];
+        if (key)
+            a['device'] = key;
+        for (const h of attrHeaders) {
+            const v = hdr(req.headers[h]);
+            if (v !== undefined)
+                a['header.' + h] = v;
+        }
+        return a;
+    };
+    // ---- traffic edges (host→service[@version]), flushed to the lens periodically ----
+    const edges = new Map();
+    const recordEdge = (route, ms, status, caller, version) => {
+        const host = route.host ?? '(default)';
+        const key = (caller ? caller + '⇒' : '') + host + '→' + route.service + (version ? '@' + version : '');
+        const e = edges.get(key) ?? { host, service: route.service, caller, version, count: 0, totalMs: 0, errors: 0 };
+        e.count += 1;
+        e.totalMs += ms;
+        if (status >= 500 || status === 0)
+            e.errors += 1;
+        edges.set(key, e);
+    };
+    const edgeFlusher = setInterval(() => {
+        if (!opts.emit) {
+            edges.clear();
+            return;
+        }
+        for (const e of edges.values()) {
+            emit(e.errors ? 'warning' : 'info', `traffic ${e.host} → ${e.service}${e.version ? '@' + e.version : ''}: ${e.count} req`, `gateway-traffic:${e.host}->${e.service}${e.version ? '@' + e.version : ''}`, { kind: 'gateway-traffic', host: e.host, project: e.service, caller: e.caller, version: e.version, count: e.count, avgMs: Math.round(e.totalMs / e.count), errors: e.errors });
+        }
+        edges.clear();
+    }, opts.trafficFlushMs ?? 5000);
+    edgeFlusher.unref?.();
+    // ---- per-request flow trace (only for flow-header-carrying requests) ----
+    const beginTrace = (req) => {
+        if (!opts.emit)
+            return undefined; // no lens → no trace overhead
+        const flowId = hdr(req.headers['x-lensmcp-flow-id']);
+        let requestId = hdr(req.headers['x-request-id']);
+        if (!flowId && !requestId && !hdr(req.headers['traceparent']))
+            return undefined;
+        if (!requestId) {
+            requestId = eid();
+            req.headers['x-request-id'] = requestId;
+        }
+        return { startedAt: Date.now(), method: (req.method ?? 'GET').toUpperCase(), host: (req.headers.host || '').split(':')[0], url: req.url ?? '/', steps: [], ...(flowId ? { flowId } : {}), ...(requestId ? { requestId } : {}) };
+    };
+    const step = (t, name, info) => { if (t)
+        t.steps.push({ step: name, atMs: Date.now() - t.startedAt, ...info }); };
+    const finish = (t, service, status, extra) => {
+        if (!t || t.done)
+            return;
+        t.done = true;
+        emit(status >= 500 ? 'error' : 'info', `gateway ${t.method} ${t.host}${t.url} → ${status} (${Date.now() - t.startedAt}ms)`, `gateway-request:${service}`, { kind: 'gateway-request', project: service, host: t.host, method: t.method, url: t.url, status, durationMs: Date.now() - t.startedAt, startedAt: t.startedAt, steps: t.steps, ...extra }, { ...(t.flowId ? { flowId: t.flowId } : {}), ...(t.requestId ? { requestId: t.requestId } : {}) });
+    };
+    rebuild(manifests); // emit initial gateway-up
+    // ---- trust boundary (verbatim dev model) ----
+    // The gateway OWNS these headers — drop any client-supplied copy at ingress so identity
+    // can only originate from a verified token (closes header spoofing), then re-stamp on forward.
+    const TRUST_HEADERS = ['x-internal-token', 'x-api-key-id', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions'];
+    const stripTrust = (req) => { for (const h of TRUST_HEADERS)
+        delete req.headers[h]; };
+    /** Stamp the verified identity onto the forwarded request — tenant from the `tid` CLAIM, never a client header. */
+    const stampIdentity = (req, claims) => {
+        const set = (k, v) => { if (v !== undefined && v !== null && v !== '')
+            req.headers[k] = String(v); };
+        set('x-user-id', claims['sub']);
+        set('x-tenant-id', claims['tid']);
+        const roles = claims['roles'];
+        if (Array.isArray(roles) && roles.length)
+            req.headers['x-user-roles'] = roles.map(String).join(',');
+        const perms = claims['perms'];
+        if (Array.isArray(perms) && perms.length)
+            req.headers['x-user-permissions'] = perms.map(String).join(',');
+    };
+    // ---- per-request upstream resolution (direct upstream OR pod pick + sticky version) ----
+    const resolveUpstream = async (route, trace, pinned) => {
+        if (route.upstream)
+            return { url: route.upstream };
+        const opt = pinned ? { version: pinned } : undefined;
+        let up = opts.pods?.pick(route.service, opt);
+        if (!up && opts.pods?.ensureUp) {
+            step(trace, 'ensure-up', { project: route.service });
+            if (await opts.pods.ensureUp(route.service))
+                up = opts.pods.pick(route.service, opt);
+        }
+        return up;
+    };
+    const reqState = new WeakMap();
+    // ---- no-route fallbacks: CORS preflight, health, discovery, 404 ----
+    const setCors = (reply, req) => {
+        reply.header('access-control-allow-origin', String(req.headers.origin ?? '*'));
+        reply.header('access-control-allow-credentials', 'true');
+        reply.header('vary', 'Origin');
+    };
+    const handleNoRoute = (request, reply, host, url) => {
+        const req = request.raw;
+        setCors(reply, req);
+        if (req.method === 'OPTIONS') {
+            reply.code(204)
+                .header('access-control-allow-methods', 'GET,POST,PUT,PATCH,DELETE,HEAD,OPTIONS')
+                .header('access-control-allow-headers', String(req.headers['access-control-request-headers'] ?? '*'))
+                .send();
+            return;
+        }
+        const mounted = routes.filter((r) => r.host && (0, manifest_1.hostMatches)(r.host, host) && r.prefix);
+        const p = url.split('?')[0];
+        if (mounted.length > 0 && (p === '/health' || p === '/healthz')) {
+            reply.code(200).type('application/json').send(JSON.stringify({ status: 'ok', host, services: mounted.map((r) => ({ service: r.service, prefix: r.prefix })) }));
+            return;
+        }
+        if (p === '/.well-known/lensmcp-routes') {
+            reply.code(200).type('application/json').send(JSON.stringify({ host, routes: routes.filter((r) => !r.host || (0, manifest_1.hostMatches)(r.host, host)).map((r) => ({ host: r.host ?? '(default)', service: r.service, prefix: r.prefix, internal: !!r.internal, auth: r.auth })) }));
+            return;
+        }
+        reply.code(404).type('text/plain; charset=utf-8').send(mounted.length > 0 ? `No service mounted at ${url} on ${host}.` : 'No gateway route for this hostname.');
+    };
+    // ---- the request handler (catch-all): route → trust → auth → forward ----
+    const handler = async (request, reply) => {
+        const req = request.raw;
+        const host = (req.headers.host || '').split(':')[0];
+        const url = req.url ?? '/';
+        const trace = beginTrace(req);
+        step(trace, 'received', { host, method: req.method, url });
+        const route = (0, manifest_1.matchRoute)(routes, host, url);
+        if (!route) {
+            handleNoRoute(request, reply, host, url);
+            return reply;
+        }
+        step(trace, 'route-match', { project: route.service, ...(route.prefix ? { prefix: route.prefix } : {}), ...(route.internal ? { internal: true } : {}) });
+        // Verify the end-user JWT ONCE (when a verifier is configured) — reused for both edge
+        // authorization and rollout targeting. Internal east-west routes use x-api-key instead.
+        const claims = route.internal ? undefined : opts.identify?.(req);
+        let caller;
+        if (route.internal) {
+            caller = keyToProject.get(String(req.headers['x-api-key'] ?? ''));
+            if (!caller) {
+                step(trace, 'auth', { mode: 'x-api-key', ok: false });
+                finish(trace, route.service, 401);
+                reply.code(401).type('text/plain').send('Invalid or missing x-api-key for internal route.');
+                return reply;
+            }
+            req.headers['x-api-key-id'] = caller;
+            delete req.headers['x-api-key'];
+            step(trace, 'auth', { mode: 'x-api-key', ok: true, caller });
+        }
+        else {
+            const m = manifestByService.get(route.service);
+            const resolved = m ? (0, manifest_1.authRuleForPath)(m, url, req.method ?? 'GET', false) : { mode: route.auth };
+            if (resolved.mode === 'jwt') {
+                if (opts.identify) {
+                    if (!claims) {
+                        step(trace, 'auth', { mode: 'jwt', ok: false });
+                        finish(trace, route.service, 401);
+                        reply.code(401).type('text/plain').send('Unauthorized: invalid or missing token');
+                        return reply;
+                    }
+                }
+                else {
+                    try {
+                        authenticate('jwt', req);
+                    }
+                    catch (e) {
+                        step(trace, 'auth', { mode: 'jwt', ok: false });
+                        finish(trace, route.service, 401);
+                        reply.code(401).type('text/plain').send(`Unauthorized: ${e.message}`);
+                        return reply;
+                    }
+                }
+                // edge ABAC: the required permission/rule must hold against the VERIFIED claims.
+                if (resolved.rule && claims && !(0, manifest_1.evalRule)(resolved.rule, buildAttrs(request, claims, undefined))) {
+                    step(trace, 'authz', { ok: false });
+                    finish(trace, route.service, 403);
+                    reply.code(403).type('text/plain').send('Forbidden: insufficient permission');
+                    return reply;
+                }
+                step(trace, 'auth', { mode: 'jwt', ok: true });
+                if (claims)
+                    stampIdentity(req, claims);
+            }
+            else {
+                step(trace, 'auth', { mode: resolved.mode });
+            }
+        }
+        if (opts.serviceKeys?.[route.service])
+            req.headers['x-internal-token'] = opts.serviceKeys[route.service];
+        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix))
+            req.url = route.prependPrefix + (req.url ?? '/');
+        // ── version selection: explicit pin → ABAC rule → sticky-key weighted → SWRR ──
+        // (targeting/stickiness only for external routes; internal east-west has no "device")
+        const explicit = hdr(req.headers[versionHeader]);
+        let pinned = explicit;
+        let mintKey; // set → append Set-Cookie on the response
+        if (!route.internal && !explicit) {
+            const sk = resolveStickyKey(req);
+            if (sk.mint)
+                mintKey = sk.key;
+            const attrs = buildAttrs(request, claims, sk.key);
+            const targeted = opts.pods?.resolveVersion?.(route.service, attrs);
+            if (targeted) {
+                pinned = targeted.version;
+                step(trace, 'targeting', { rule: targeted.rule, version: targeted.version });
+            }
+            else if (sk.key) {
+                const b = opts.pods?.bucket?.(route.service, sk.key);
+                if (b) {
+                    pinned = b.version;
+                    step(trace, 'targeting', { mode: 'sticky', version: b.version });
+                }
+            }
+        }
+        const up = await resolveUpstream(route, trace, pinned);
+        if (!up) {
+            finish(trace, route.service, 503);
+            reply.code(503).type('text/plain').send(`Service ${route.service} has no available upstream.`);
+            return reply;
+        }
+        if (!('url' in up)) {
+            finish(trace, route.service, 502);
+            reply.code(502).type('text/plain').send(`Service ${route.service}: socket upstreams are not supported by the prod gateway.`);
+            return reply;
+        }
+        const origin = new URL(up.url).origin;
+        const upLabel = up.url;
+        reqState.set(request, { route, caller, startedAt: Date.now(), trace, version: up.version, upLabel });
+        step(trace, up.version ? 'rollout' : 'upstream', { upstream: upLabel, ...(up.version ? { version: up.version, ...(explicit ? { pinned: true } : {}) } : {}) });
+        // No source → reply-from forwards the (prefix-rewritten) req.url to `origin`.
+        return reply.from(undefined, {
+            getUpstream: () => origin,
+            rewriteHeaders: (headers) => {
+                headers['x-lensmcp-upstream'] = upLabel;
+                if (up.version)
+                    headers['x-lensmcp-version'] = up.version; // which rollout cohort served
+                // The gateway owns the edge response: drop the upstream framework leak
+                // (e.g. NestJS/Express `x-powered-by`) and brand the hop as the LensMCP gateway.
+                delete headers['x-powered-by'];
+                headers['server'] = 'LensMCP';
+                if (mintKey) { // first contact → stamp the device cookie (appended to any upstream Set-Cookie)
+                    const ex = headers['set-cookie'];
+                    const list = Array.isArray(ex) ? ex : ex ? [ex] : [];
+                    headers['set-cookie'] = [...list, mintCookieHeader(mintKey)];
+                }
+                return headers;
+            },
+            onError: (rep, { error }) => {
+                opts.pods?.drop?.(route.service, up);
+                rep.code(502).type('text/plain').send(`Upstream ${route.service} unavailable — ${error.message}`);
+            },
+        });
+    };
+    // ---- raw WebSocket upgrade proxy (same trust model; net-level pipe) ----
+    const upgrade = (req, socket, head) => {
+        stripTrust(req);
+        const host = (req.headers.host || '').split(':')[0];
+        const route = (0, manifest_1.matchRoute)(routes, host, req.url ?? '/');
+        if (!route) {
+            socket.destroy();
+            return;
+        }
+        const claims = route.internal ? undefined : opts.identify?.(req);
+        let caller;
+        if (route.internal) {
+            caller = keyToProject.get(String(req.headers['x-api-key'] ?? ''));
+            if (!caller) {
+                socket.destroy();
+                return;
+            }
+        }
+        else {
+            const m = manifestByService.get(route.service);
+            const resolved = m ? (0, manifest_1.authRuleForPath)(m, req.url ?? '/', req.method ?? 'GET', false) : { mode: route.auth };
+            if (resolved.mode === 'jwt') {
+                if (opts.identify) {
+                    if (!claims) {
+                        socket.destroy();
+                        return;
+                    }
+                }
+                else {
+                    try {
+                        authenticate('jwt', req);
+                    }
+                    catch {
+                        socket.destroy();
+                        return;
+                    }
+                }
+                if (resolved.rule && claims && !(0, manifest_1.evalRule)(resolved.rule, { ...claims, ip: req.socket.remoteAddress ?? '', method: req.method ?? 'GET', path: (req.url ?? '/').split('?')[0] })) {
+                    socket.destroy();
+                    return;
+                }
+            }
+        }
+        if (route.prependPrefix && !req.url?.startsWith(route.prependPrefix))
+            req.url = route.prependPrefix + (req.url ?? '/');
+        const pinned = hdr(req.headers[versionHeader]);
+        void (async () => {
+            const up = await resolveUpstream(route, undefined, pinned);
+            if (!up || !('url' in up)) {
+                socket.destroy();
+                return;
+            }
+            const u = new URL(up.url);
+            const upstream = net.connect(Number(u.port) || (u.protocol === 'https:' ? 443 : 80), u.hostname, () => {
+                // rebuild the upgrade request from rawHeaders (verbatim), minus the trust headers
+                // we own (incl. client identity), plus the validated caller + target token + identity.
+                const strip = new Set(['x-internal-token', 'x-api-key-id', 'x-api-key', 'x-user-id', 'x-tenant-id', 'x-user-roles', 'x-user-permissions']);
+                const lines = [`${req.method} ${req.url} HTTP/1.1`];
+                for (let i = 0; i < req.rawHeaders.length; i += 2) {
+                    const k = req.rawHeaders[i];
+                    if (!strip.has(k.toLowerCase()))
+                        lines.push(`${k}: ${req.rawHeaders[i + 1]}`);
+                }
+                if (caller)
+                    lines.push(`x-api-key-id: ${caller}`);
+                const token = opts.serviceKeys?.[route.service];
+                if (token)
+                    lines.push(`x-internal-token: ${token}`);
+                if (claims) { // stamp the verified identity (mirrors the HTTP path's stampIdentity)
+                    if (claims['sub'])
+                        lines.push(`x-user-id: ${String(claims['sub'])}`);
+                    if (claims['tid'])
+                        lines.push(`x-tenant-id: ${String(claims['tid'])}`);
+                    const roles = claims['roles'];
+                    if (Array.isArray(roles) && roles.length)
+                        lines.push(`x-user-roles: ${roles.map(String).join(',')}`);
+                    const perms = claims['perms'];
+                    if (Array.isArray(perms) && perms.length)
+                        lines.push(`x-user-permissions: ${perms.map(String).join(',')}`);
+                }
+                upstream.write(lines.join('\r\n') + '\r\n\r\n');
+                if (head?.length)
+                    upstream.write(head);
+                socket.pipe(upstream);
+                upstream.pipe(socket);
+            });
+            upstream.on('error', () => socket.destroy());
+            socket.on('error', () => upstream.destroy());
+        })();
+    };
+    // ---- build one Fastify app per port (shared route table + providers) ----
+    const buildApp = async () => {
+        const app = (0, fastify_1.default)({ logger: false, ...(opts.trustProxyIp ? { trustProxy: true } : {}), ...(opts.tls ? { https: { key: opts.tls.key, cert: opts.tls.cert } } : {}) });
+        // Stream every body straight through — never parse/buffer (this is a proxy).
+        app.removeAllContentTypeParsers();
+        app.addContentTypeParser('*', (_req, payload, done) => done(null, payload));
+        await app.register(reply_from_1.default, {
+            disableRequestLogging: true,
+            undici: {
+                connections: opts.undici?.connections ?? 256,
+                pipelining: opts.undici?.pipelining ?? 1,
+                keepAliveTimeout: opts.undici?.keepAliveTimeout ?? 60_000,
+                connect: { rejectUnauthorized: false }, // upstreams may use the local CA / self-signed (mirrors http-proxy secure:false)
+            },
+        });
+        // Host-agnostic operational probes (answered before routing).
+        app.get('/livez', (_req, reply) => { reply.type('text/plain').send('ok'); });
+        app.get('/readyz', (_req, reply) => {
+            const ready = routes.length > 0;
+            reply.code(ready ? 200 : 503).type('text/plain').send(ready ? 'ready' : 'no routes loaded');
+        });
+        app.addHook('onRequest', (request, _reply, done) => { stripTrust(request.raw); done(); });
+        app.addHook('onResponse', (request, reply, done) => {
+            const st = reqState.get(request);
+            if (st) {
+                recordEdge(st.route, Date.now() - st.startedAt, reply.statusCode ?? 0, st.caller, st.version);
+                finish(st.trace, st.route.service, reply.statusCode ?? 0, { upstream: st.upLabel, ...(st.version ? { version: st.version } : {}), ...(st.caller ? { caller: st.caller } : {}) });
+                reqState.delete(request);
+            }
+            done();
+        });
+        app.all('/', handler);
+        app.all('/*', handler);
+        app.server.on('upgrade', upgrade);
+        return app;
+    };
+    // ---- listeners ----
+    const apps = [];
+    const boundPorts = [];
+    for (const port of ports) {
+        const app = await buildApp();
+        await app.listen({ port, host: '0.0.0.0' });
+        const addr = app.server.address();
+        const bound = typeof addr === 'object' && addr ? addr.port : port;
+        boundPorts.push(bound);
+        console.log(`[gateway] listening on ${opts.tls ? 'https' : 'http'}://0.0.0.0:${bound}`);
+        apps.push(app);
+    }
+    let stopped = false;
+    return {
+        ports: boundPorts,
+        routes: () => routes,
+        stop: async () => {
+            if (stopped)
+                return;
+            stopped = true;
+            clearInterval(edgeFlusher);
+            unwatch?.();
+            emit('info', 'gateway down', 'gateway-down', { kind: 'gateway-down' });
+            await Promise.all(apps.map((a) => a.close())); // reply-from closes its undici pool on close
+        },
+    };
+}

package/executors/gateway/providers-prod.d.ts

@@ -0,0 +1,46 @@
+import { type ManifestProvider, type PodProvider, type RouteManifest, type VersionCohort, type TargetRule } from './manifest';
+/** A service's rollout spec: optional targeting rules + the weighted cohorts. */
+export interface RolloutSpec {
+    rules?: TargetRule[];
+    cohorts: VersionCohort[];
+}
+/** Per-service rollout table — a bare cohort array (no rules) or a full spec. */
+export type RolloutTable = Record<string, VersionCohort[] | RolloutSpec>;
+/** A fixed set of manifests (tests, single-shot config). */
+export declare function staticManifestProvider(manifests: RouteManifest[]): ManifestProvider;
+/**
+ * Watch a manifests JSON file; atomically swap the in-memory set on a valid
+ * change, reject and keep last-good on a bad one. `watch` notifies subscribers
+ * with the new set. Poll fallback covers editors/atomic-rename + platforms
+ * where fs.watch is unreliable.
+ */
+export declare function fileManifestProvider(filePath: string, opts?: {
+    pollMs?: number;
+    onError?: (e: unknown) => void;
+}): ManifestProvider & {
+    stop: () => void;
+};
+/**
+ * Rollout-aware pod provider: each service is a set of VERSION COHORTS, and
+ * `pick` splits traffic across them by weight (smooth weighted round-robin —
+ * a 90/10 canary interleaves, never bursts) and round-robins endpoints WITHIN
+ * the chosen cohort. `pick(svc, {version})` pins a cohort (sticky canary /
+ * debugging). `update` swaps weights/cohorts live — a rollout ramps
+ * (10→25→50→100) or rolls back without redeploying the gateway. The returned
+ * Upstream carries `version` so the gateway can surface which cohort served.
+ */
+export declare function rolloutPodProvider(table: RolloutTable): PodProvider & {
+    update: (next: RolloutTable) => void;
+};
+/**
+ * Flat single-version load balancing — the common case (no rollout in
+ * progress). A thin wrapper over `rolloutPodProvider` with one synthetic
+ * cohort per service, so the rollout path is the only code path. The synthetic
+ * version is STRIPPED from the result: flat LB carries no `version`, so a
+ * response only gets `x-lensmcp-version` (and a `rollout` trace step) when a
+ * genuine multi-cohort rollout served it.
+ */
+export declare function endpointsPodProvider(endpoints: Record<string, string[]>): PodProvider & {
+    update: (next: Record<string, string[]>) => void;
+};
+//# sourceMappingURL=providers-prod.d.ts.map

package/executors/gateway/providers-prod.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers-prod.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/providers-prod.ts"],"names":[],"mappings":"AAkBA,OAAO,EAEL,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAiB,KAAK,aAAa,EAAE,KAAK,UAAU,EAChH,MAAM,YAAY,CAAC;AAEpB,iFAAiF;AACjF,MAAM,WAAW,WAAW;IAAG,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE;AAC/E,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,WAAW,CAAC,CAAC;AAIzE,4DAA4D;AAC5D,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAEnF;AAgBD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAA;CAAO,GAC7D,gBAAgB,GAAG;IAAE,IAAI,EAAE,MAAM,IAAI,CAAA;CAAE,CA+BzC;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,YAAY,GAClB,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,KAAK,IAAI,CAAA;CAAE,CAkDxD;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAClC,WAAW,GAAG;IAAE,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,KAAK,IAAI,CAAA;CAAE,CAapE"}