@lensmcp/cluster 1.0.0 → 1.2.0

package/executors/gateway/otel-tracing.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setupTracing = setupTracing;
+/**
+ * OpenTelemetry tracing for the production gateway — OPT-IN, cloud-neutral.
+ *
+ * Makes the gateway a first-class span in a distributed trace: it continues the
+ * incoming W3C `traceparent`, emits one SERVER span per request, and INJECTS its
+ * context into the forwarded headers so the downstream service's spans parent to
+ * the gateway hop. Spans export over OTLP/HTTP — point `OTEL_EXPORTER_OTLP_ENDPOINT`
+ * at Cloud Trace (GCP), the OTel Collector / X-Ray (AWS), Tempo, Jaeger, etc. The
+ * SAME image traces on any cloud; only the endpoint env changes.
+ *
+ * All OpenTelemetry imports live HERE. The gateway lib takes a `GatewayTracer`
+ * (type-only) and calls these helpers, so it never pulls OTel unless tracing is
+ * wired — and when no endpoint is set, `setupTracing` returns undefined and the
+ * hot path does zero span work.
+ *
+ * No global context manager (`provider.register()` is intentionally NOT called):
+ * context is threaded explicitly via extract/inject, keeping the proxy hot path
+ * synchronous and free of async-hooks overhead.
+ */
+const api_1 = require("@opentelemetry/api");
+const core_1 = require("@opentelemetry/core");
+const sdk_trace_base_1 = require("@opentelemetry/sdk-trace-base");
+const resources_1 = require("@opentelemetry/resources");
+const exporter_trace_otlp_http_1 = require("@opentelemetry/exporter-trace-otlp-http");
+const firstHeader = (h, k) => {
+    const v = h[k.toLowerCase()];
+    return Array.isArray(v) ? v[0] : v;
+};
+const getter = {
+    keys: (carrier) => Object.keys(carrier),
+    get: (carrier, key) => firstHeader(carrier, key),
+};
+const setter = {
+    set: (carrier, key, value) => { carrier[key.toLowerCase()] = value; },
+};
+/** Build the gateway tracer, or undefined when no OTLP endpoint is configured
+ *  (default-off → the lib skips all span work, zero hot-path overhead). */
+function setupTracing(opts = {}) {
+    const endpoint = opts.endpoint ?? process.env['OTEL_EXPORTER_OTLP_ENDPOINT'];
+    if (!endpoint)
+        return undefined;
+    const base = endpoint.replace(/\/+$/, '');
+    const url = /\/v1\/traces$/.test(base) ? base : `${base}/v1/traces`;
+    const exporter = new exporter_trace_otlp_http_1.OTLPTraceExporter({ url, ...(opts.headers ? { headers: opts.headers } : {}) });
+    const provider = new sdk_trace_base_1.BasicTracerProvider({
+        resource: new resources_1.Resource({ 'service.name': opts.serviceName ?? process.env['OTEL_SERVICE_NAME'] ?? 'lensmcp-gateway' }),
+        sampler: new sdk_trace_base_1.ParentBasedSampler({ root: new sdk_trace_base_1.TraceIdRatioBasedSampler(opts.sampleRatio ?? 1) }),
+    });
+    provider.addSpanProcessor(new sdk_trace_base_1.BatchSpanProcessor(exporter));
+    const tracer = provider.getTracer('lensmcp-gateway');
+    const propagator = new core_1.W3CTraceContextPropagator();
+    return {
+        startSpan(req, name, attrs) {
+            const parent = propagator.extract(api_1.ROOT_CONTEXT, req.headers, getter);
+            return tracer.startSpan(name, { kind: api_1.SpanKind.SERVER, attributes: attrs }, parent);
+        },
+        inject(span, headers) {
+            propagator.inject(api_1.trace.setSpan(api_1.ROOT_CONTEXT, span), headers, setter);
+        },
+        end(span, statusCode, attrs) {
+            if (attrs)
+                span.setAttributes(attrs);
+            span.setAttribute('http.response.status_code', statusCode);
+            if (statusCode >= 500 || statusCode === 0)
+                span.setStatus({ code: api_1.SpanStatusCode.ERROR });
+            span.end();
+        },
+        shutdown: () => provider.shutdown(),
+    };
+}

package/executors/gateway/prod-gateway.lib.d.ts

@@ -1,5 +1,22 @@
 import * as http from 'node:http';
 import { type AuthMode, type ManifestProvider, type PodProvider, type Route } from './manifest';
+import type { GatewayTracer } from './otel-tracing';
+import type { HealthChecker } from './health-check';
+import type { RateLimiter } from './rate-limit';
+import type { ProcessMetrics } from './metrics';
+/** Per-service health rollup, derived from observed traffic over a rolling window.
+ *  `status` is inferred from the recent error rate (passive — not an active probe):
+ *  healthy <5% err · degraded 5–50% · down ≥50% · idle (no recent traffic). */
+export interface ServiceStatus {
+    service: string;
+    hosts: string[];
+    status: 'healthy' | 'degraded' | 'down' | 'idle';
+    requests: number;
+    errorRate: number;
+    avgMs: number;
+    versions: string[];
+    lastSeen: number | null;
+}
 export interface ProdGatewayOptions {
     /** Ports to bind. Default `[8080]` (or `[8443]` with TLS). One Fastify
      *  instance per port; all share the route table + providers. */
@@ -23,12 +40,80 @@ export interface ProdGatewayOptions {
     /** Request header that PINS a rollout cohort (sticky canary / debugging).
      *  Default `x-lensmcp-version`. The chosen version is echoed on the response. */
     versionHeader?: string;
-    /** undici pool tuning passed to reply-from (connections, pipelining…). */
+    /** undici pool tuning passed to reply-from. `bodyTimeout: 0` (default) keeps
+     *  long-lived SSE / streamable-HTTP MCP responses open — undici's 300s default
+     *  would otherwise abort an idle stream. `rejectUnauthorized` defaults false
+     *  (local-CA / self-signed upstreams); set true for verified east-west on GCP.
+     *  `allowH2` multiplexes many concurrent streams over few HTTP/2 connections
+     *  (Cloud Run supports h2 end-to-end) — useful when one upstream fans many MCP streams.
+     *  `cert`/`key` present a client certificate to upstreams (mTLS); `ca` trusts a
+     *  private CA for the server side. `rejectUnauthorized:false` keeps mTLS while
+     *  skipping server-cert validation (self-signed internal services). */
     undici?: {
         connections?: number;
         pipelining?: number;
         keepAliveTimeout?: number;
+        bodyTimeout?: number;
+        headersTimeout?: number;
+        rejectUnauthorized?: boolean;
+        allowH2?: boolean;
+        cert?: string | Buffer;
+        key?: string | Buffer;
+        ca?: string | Buffer;
+    };
+    /** Header carrying the AUTHORITATIVE client IP from a trusted front proxy
+     *  (e.g. `cf-connecting-ip` behind Cloudflare). When set, the `ip` ABAC
+     *  attribute comes from this header instead of the (multi-proxy, spoofable)
+     *  X-Forwarded-For chain. Safe only when the origin is locked to that proxy —
+     *  which Cloudflare Tunnel guarantees (there is no public origin to bypass). */
+    clientIpHeader?: string;
+    /** CIDRs of trusted front proxies. `clientIpHeader` is honored ONLY when the
+     *  immediate peer (socket address) is in this set — otherwise the header is
+     *  ignored and the socket IP is used (prevents spoofing the `ip` ABAC attribute
+     *  by hitting the origin directly). Default when omitted: loopback only
+     *  (matches a co-located cloudflared/sidecar). */
+    clientIpTrustedProxies?: string[];
+    /** OpenTelemetry tracer (from `setupTracing`). When set, each request gets a
+     *  SERVER span whose context is propagated downstream (W3C `traceparent`), so
+     *  the gateway hop stitches into the distributed trace. Omit → no span work. */
+    tracing?: GatewayTracer;
+    /** Expose `GET /statusz` — the per-service health rollup as JSON. Requires
+     *  `statusToken` too (fail-closed: served only when both are set). Default off (404). */
+    statusEndpoint?: boolean;
+    /** Token `/statusz` requires (via `?token=` or `x-status-token`), compared in
+     *  constant time. REQUIRED for the endpoint to serve — no token → /statusz is 404. */
+    statusToken?: string;
+    /** How many flush windows the rolling status sums over (default 12 → ~60s at the 5s flush). */
+    statusWindows?: number;
+    /** Called each flush window with the per-service status snapshot — wire it to a
+     *  Redis `status:*` fan-out / status page. The gateway is the only thing that
+     *  sees every request, so this is the natural source of observed health. */
+    onStatus?: (snapshot: ServiceStatus[]) => void;
+    /** Active upstream health-checker (from `createHealthChecker`). When set, the
+     *  gateway probes upstreams, steers pooled picks toward healthy endpoints, and
+     *  reflects true liveness in /statusz. Omit → passive drop-on-error only. */
+    healthChecker?: HealthChecker;
+    /** Opt-in per-client-IP rate limiter — defense-in-depth behind the edge throttle.
+     *  In-memory + per-instance (`createRateLimiter`) or ONE global budget across all
+     *  containers (`createRedisRateLimiter`). The check is `await`ed either way; it
+     *  runs before routing/auth and applies only to proxied traffic (ops endpoints exempt). */
+    rateLimiter?: RateLimiter;
+    /** Process self-metrics (event-loop lag / rss / cpu / req rates) from `createProcessMetrics`;
+     *  exposed at the token-gated `/metricsz` for load/stress tests + production monitoring. */
+    metrics?: ProcessMetrics;
+    /** Server-side HTTP timeouts (ms). `keepAliveTimeout` default 620000 — kept
+     *  ABOVE the GCP external ALB's 600s backend keep-alive so the gateway never
+     *  closes a pooled connection the LB still considers live (the classic
+     *  intermittent-502 race). `requestTimeout` default 0 (disabled) for streaming. */
+    server?: {
+        keepAliveTimeout?: number;
+        requestTimeout?: number;
+        connectionTimeout?: number;
+        headersTimeout?: number;
     };
+    /** Opt-in one-JSON-line-per-request access log to stdout, shaped for Cloud
+     *  Logging (severity + time + message). Default off (zero overhead). */
+    accessLog?: boolean;
     /** Return verified subject attributes (e.g. JWT claims) for ABAC routing.
      *  MUST verify (signature/exp) — forged claims must not reach the rules. */
     identify?: (req: http.IncomingMessage) => Record<string, unknown> | undefined;
@@ -52,6 +137,7 @@ export interface ProdGatewayOptions {
 export interface ProdGatewayHandle {
     ports: number[];
     routes: () => Route[];
+    status: () => ServiceStatus[];
     stop: () => Promise<void>;
 }
 export declare function startProdGateway(opts: ProdGatewayOptions): Promise<ProdGatewayHandle>;

package/executors/gateway/prod-gateway.lib.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prod-gateway.lib.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/prod-gateway.lib.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EAEL,KAAK,QAAQ,EAAc,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAqB,KAAK,KAAK,EAClG,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,kBAAkB;IACjC;oEACgE;IAChE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,2EAA2E;IAC3E,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC;qFACiF;IACjF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACnE,GAAG,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACtD,8EAA8E;IAC9E,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChD,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;qFACiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,MAAM,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAGlF;gFAC4E;IAC5E,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9E,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;4EACwE;IACxE,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9E;sEACkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,KAAK,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAYD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoY3F"}
+{"version":3,"file":"prod-gateway.lib.d.ts","sourceRoot":"","sources":["../../../../../libs/cluster/src/executors/gateway/prod-gateway.lib.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAIlC,OAAO,EAEL,KAAK,QAAQ,EAAc,KAAK,gBAAgB,EAAE,KAAK,WAAW,EAAqB,KAAK,KAAK,EAClG,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,aAAa,EAAa,MAAM,gBAAgB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAEhD;;+EAE+E;AAC/E,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IACjD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC;oEACgE;IAChE,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,2EAA2E;IAC3E,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC;qFACiF;IACjF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACnE,GAAG,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACtD,8EAA8E;IAC9E,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;IAChD,oDAAoD;IACpD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;qFACiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;2EAQuE;IACvE,MAAM,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACvP;;;;oFAIgF;IAChF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;sDAIkD;IAClD,sBAAsB,CAAC,EAAE,MAAM,EAAE,CAAC;IAClC;;oFAEgF;IAChF,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB;6FACyF;IACzF,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;0FACsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+FAA+F;IAC/F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;gFAE4E;IAC5E,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,IAAI,CAAC;IAC/C;;iFAE6E;IAC7E,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;+FAG2F;IAC3F,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;gGAC4F;IAC5F,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB;;;uFAGmF;IACnF,MAAM,CAAC,EAAE;QAAE,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrH;4EACwE;IACxE,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB;gFAC4E;IAC5E,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC9E,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;4EACwE;IACxE,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9E;sEACkE;IAClE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,EAAE,MAAM,KAAK,EAAE,CAAC;IACtB,MAAM,EAAE,MAAM,aAAa,EAAE,CAAC;IAC9B,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AA+BD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA0kB3F"}