@atomic-chat/core 0.5.1 → 0.13.2

package/dist/runtime/managed-text/gateway.d.ts

@@ -0,0 +1,158 @@
+/**
+ * The per-session authenticating gateway that sits in front of a managed engine's port.
+ *
+ * `trtllm-serve` (and any future container-hosted engine) listens on loopback but checks no API key
+ * of its own, so anything on the machine — and, through DNS rebinding, any web page the user has
+ * open — could reach it directly. Core instead binds a listener of its own on `127.0.0.1:<random>`
+ * for each session, hands that port and a fresh key out as `SessionInfo.port`/`api_key`, and proxies
+ * through to the container's real port only once a request presents that key from an allowed Host.
+ * The container's own port is never handed to a caller.
+ *
+ * The Host and Bearer-key checks are the public server's own (`server/public/gates.ts`,
+ * `hostAndKeyGate`) — the same code `:1337` gates on, so the trusted-Host allowlist behaves
+ * identically everywhere in the process. Streaming is the public server's own upstream plumbing too
+ * (`server/public/wire.ts`): `sendUpstream` for the request, `relay`/`pipeBody` to copy the response
+ * to the client as it arrives, with no buffering, and to close the upstream connection the moment
+ * the client goes away.
+ *
+ * New for TensorRT-LLM managed sessions (design D11, spec `managed-session-gateway`). Engine-neutral:
+ * nothing here knows about `trtllm-serve` specifically, only that it speaks HTTP on loopback.
+ *
+ * `routes`/`rewritableRoutes`/`rewriteRequestBody` (task 2.13 fix rounds 1-3; ADR
+ * `docs/decisions/2026-09-28-tensorrt-llm-output-cap-enforced-by-the-session-gateway.md`) are the
+ * deliberate exceptions to design D11's own Risks/Trade-offs entry, "только копирование байтов без
+ * парсинга" (only copies bytes, never parses):
+ * - every request's method+path is checked against the adapter's own declared `routes` before
+ *   anything is forwarded — a path the adapter never declared at all gets `404`; a path it declared
+ *   for a *different* method gets `405` with an `Allow` header (round 3: round 2's path-only
+ *   matching let a wrong method through). Neither is ever forwarded, closing off both whatever else
+ *   the engine's own HTTP server happens to also expose, and a wrong method on a route that is
+ *   otherwise real;
+ * - a request matching a declared route the adapter also lists in `rewritableRoutes` gets its JSON
+ *   body parsed, handed to `rewriteRequestBody`, and re-serialized before forwarding.
+ * Every other declared route still proxies byte-for-byte, exactly as before this existed. This
+ * module stays engine-neutral about *why* an adapter wants either of these — it only enforces the
+ * mechanics (method+path route matching on the decoded path, a body-size cap enforced while the body
+ * streams in, a parse failure or a rewriter's own throw answering `400`/`500` instead of forwarding).
+ *
+ * `mapErrorResponse` (task 2.14 fix round 1, findings-2.14-r1.md item 1; ADR
+ * `docs/decisions/2026-09-29-tensorrt-llm-sessions-carry-a-route-policy-to-the-public-server.md`) is the
+ * one exception on the response side, and only for errors: a non-2xx answer on a declared POST route
+ * is read (up to `MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES`) and answered with whatever OpenAI error the
+ * adapter maps it to, so a client talking to `SessionInfo.port` directly gets the same
+ * `context_length_exceeded` as one going through `:1337`. A 2xx answer is never read or parsed: it
+ * streams through as it arrives, exactly as before.
+ */
+import type { ManagedResponseRewrite, ManagedRoute } from './adapter.js';
+/**
+ * `host`, bracketed when it is a bare IPv6 literal — what has to follow `http://` for `new URL` (and
+ * everything downstream of it) to parse the address instead of tripping over its colons. `::1` alone
+ * would otherwise be read as `host "::1"` with a bogus port after the last colon; `[::1]` already
+ * carries its own brackets and is left alone, as is every other accepted loopback spelling.
+ */
+export declare function bracketIfIpv6(host: string): string;
+/** The container port the gateway proxies to. Always loopback: never a LAN or public address. */
+export interface ManagedGatewayUpstream {
+    host: string;
+    port: number;
+}
+export interface ManagedGatewayOptions {
+    upstream: ManagedGatewayUpstream;
+    /** Bearer key clients must present; unique per session generation, see {@link generateGatewayKey}. */
+    apiKey: string;
+    /** The same trusted-hosts list the public server (`:1337`) is configured with. */
+    allowedHosts: string[];
+    /**
+     * The adapter's own declared method+path routes (`ManagedTextAdapter.routes`). Matched against
+     * the request's method and percent-decoded path — see `decodedRoute` — before anything is
+     * forwarded: no declared route for the path at all is `404`; the path is declared, but not for
+     * this method, is `405` with an `Allow` header (findings-2.13-r3.md item 1). Neither reaches the
+     * upstream.
+     */
+    routes: readonly ManagedRoute[];
+    /** The subset of `routes` `rewriteRequestBody` may rewrite (`ManagedTextAdapter.rewritableRoutes`).
+     *  Absent or empty: nothing is ever parsed as JSON, even with `rewriteRequestBody` set. */
+    rewritableRoutes?: readonly ManagedRoute[];
+    /**
+     * Optional: rewrites a POST request's parsed JSON body before it is forwarded, already bound to
+     * whatever settings the adapter needs (the lifecycle does that binding; this callback takes just
+     * `route` and the parsed `body`). Called only for a route listed in `rewritableRoutes`. Absent,
+     * every request proxies exactly as before. See the file header and
+     * `ManagedTextAdapter.rewriteRequestBody`.
+     */
+    rewriteRequestBody?: (route: string, body: unknown) => unknown;
+    /** `ManagedTextAdapter.rewriteResponseFor`, bound to the session's family; rewritable routes only. */
+    rewriteResponseFor?: (route: string, requestBody: unknown) => ManagedResponseRewrite | null;
+    /**
+     * Optional: the OpenAI error body for an engine's non-2xx answer on a declared POST route, or null
+     * to relay it as it is (`ManagedTextAdapter.mapErrorResponse`). Never called for a 2xx answer.
+     */
+    mapErrorResponse?: (route: string, status: number, body: string) => object | null;
+    /**
+     * Optional: why the engine cut a 2xx event stream short, for the error event the gateway sends in
+     * its place (the session's log tail read by the adapter, `ManagedTextAdapter.describeStreamFailure`);
+     * null for no better words than the gateway's own.
+     */
+    streamFailure?: () => Promise<string | null>;
+}
+export interface ManagedGateway {
+    /** Always `127.0.0.1`: the gateway never binds anything else. */
+    host: string;
+    /** The gateway's own loopback port; this, not `upstream.port`, is what `SessionInfo.port` gets. */
+    port: number;
+    /** Stops accepting connections and drops whatever is in flight; no drain, no deferral. */
+    close: () => Promise<void>;
+}
+/**
+ * The largest request body the rewrite path will read at all. Enforced *while the body streams in*
+ * (`readCappedBody`, below), not after it is fully buffered: reading stops the moment the running
+ * total crosses the cap, so an oversized body is never held in memory in full just to then be thrown
+ * away (findings-2.13-r2.md item 2). A declared route that is not rewritable never goes through this
+ * cap at all; it proxies through the same unbounded `readBody` every request used before any of this
+ * existed.
+ */
+export declare const MANAGED_GATEWAY_REWRITE_BODY_CAP_BYTES: number;
+/**
+ * The largest engine *error* answer `mapErrorResponse` gets to read. An engine error is a short JSON
+ * document; one past this is answered with a generic error of the same status instead of being held
+ * in memory (findings-2.14-r1.md item 1: bounded, and only ever for non-2xx answers).
+ */
+export declare const MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES: number;
+/**
+ * Reads a body up to `capBytes` from any chunked async source (an `IncomingMessage` satisfies
+ * `AsyncIterable<Buffer>` on its own), counting while chunks arrive. The moment the running total
+ * would exceed the cap, reading stops immediately — the source is never asked for another chunk,
+ * and nothing past the cap is ever buffered — and `'too-large'` comes back instead of a `Buffer`
+ * (findings-2.13-r2.md item 2: exported, and typed against the generic interface rather than
+ * `IncomingMessage` specifically, so this exact claim — stops pulling before the cap, not after —
+ * has a direct unit test against a source that never ends, instead of only an HTTP-level test whose
+ * client would have to keep sending past a connection the server has already closed, which is an
+ * inherently racy thing to assert on at the TCP level).
+ *
+ * That early `return` is not as passive as it looks (findings-2.13-r3.md item 4): exiting a `for
+ * await...of` loop before it runs to completion — by `return`, `break` or an uncaught throw — makes
+ * the language call the async iterator's own `.return()`, and for a Node.js `Readable` (which is
+ * what a real `IncomingMessage` is), that already tears the stream down as a side effect, before
+ * this function's caller ever gets a chance to decide anything. So for the real gateway path, the
+ * connection's read side is already gone by the time `readCappedBody` returns `'too-large'` — this
+ * function does not *decide* to leave the connection alone, the language leaves it no choice in the
+ * matter. (This has no such effect on a `source` that is not backed by a Node stream — an async
+ * generator, say, as this file's own unit test for this function uses — where "stop iterating" truly
+ * is just that.) `sendTooLargeAndClose` still calls `req.destroy()` itself explicitly, both because
+ * destroying an already-destroyed stream is a harmless no-op and because relying on this implicit
+ * side effect would silently stop working for a source that is not a plain Node stream.
+ */
+export declare function readCappedBody(source: AsyncIterable<Buffer>, capBytes: number): Promise<Buffer | 'too-large'>;
+/**
+ * A fresh, unguessable session key: 256 bits of `crypto.randomBytes`, base64url so it fits a Bearer
+ * header verbatim. Call this once per generation — a reload gets a new key, so a caller holding the
+ * previous one is refused (`SESSION_GENERATION_STALE` at the session layer; here, plain 401).
+ */
+export declare function generateGatewayKey(): string;
+/** What a client reads when the engine ends a stream without finishing it and says nothing better. */
+export declare const MANAGED_STREAM_CUT_MESSAGE = "The engine stopped the answer before finishing it.";
+/**
+ * Start a gateway for one managed session: binds `127.0.0.1:0`, gates every request on the Host and
+ * Bearer checks, and streams whatever passes through to `options.upstream`.
+ */
+export declare function startManagedGateway(options: ManagedGatewayOptions): Promise<ManagedGateway>;