@atomic-chat/core 0.5.1 → 0.13.2

package/dist/runtime/managed-text/gateway.js

@@ -0,0 +1,535 @@
+/**
+ * The per-session authenticating gateway that sits in front of a managed engine's port.
+ *
+ * `trtllm-serve` (and any future container-hosted engine) listens on loopback but checks no API key
+ * of its own, so anything on the machine — and, through DNS rebinding, any web page the user has
+ * open — could reach it directly. Core instead binds a listener of its own on `127.0.0.1:<random>`
+ * for each session, hands that port and a fresh key out as `SessionInfo.port`/`api_key`, and proxies
+ * through to the container's real port only once a request presents that key from an allowed Host.
+ * The container's own port is never handed to a caller.
+ *
+ * The Host and Bearer-key checks are the public server's own (`server/public/gates.ts`,
+ * `hostAndKeyGate`) — the same code `:1337` gates on, so the trusted-Host allowlist behaves
+ * identically everywhere in the process. Streaming is the public server's own upstream plumbing too
+ * (`server/public/wire.ts`): `sendUpstream` for the request, `relay`/`pipeBody` to copy the response
+ * to the client as it arrives, with no buffering, and to close the upstream connection the moment
+ * the client goes away.
+ *
+ * New for TensorRT-LLM managed sessions (design D11, spec `managed-session-gateway`). Engine-neutral:
+ * nothing here knows about `trtllm-serve` specifically, only that it speaks HTTP on loopback.
+ *
+ * `routes`/`rewritableRoutes`/`rewriteRequestBody` (task 2.13 fix rounds 1-3; ADR
+ * `docs/decisions/2026-09-28-tensorrt-llm-output-cap-enforced-by-the-session-gateway.md`) are the
+ * deliberate exceptions to design D11's own Risks/Trade-offs entry, "только копирование байтов без
+ * парсинга" (only copies bytes, never parses):
+ * - every request's method+path is checked against the adapter's own declared `routes` before
+ *   anything is forwarded — a path the adapter never declared at all gets `404`; a path it declared
+ *   for a *different* method gets `405` with an `Allow` header (round 3: round 2's path-only
+ *   matching let a wrong method through). Neither is ever forwarded, closing off both whatever else
+ *   the engine's own HTTP server happens to also expose, and a wrong method on a route that is
+ *   otherwise real;
+ * - a request matching a declared route the adapter also lists in `rewritableRoutes` gets its JSON
+ *   body parsed, handed to `rewriteRequestBody`, and re-serialized before forwarding.
+ * Every other declared route still proxies byte-for-byte, exactly as before this existed. This
+ * module stays engine-neutral about *why* an adapter wants either of these — it only enforces the
+ * mechanics (method+path route matching on the decoded path, a body-size cap enforced while the body
+ * streams in, a parse failure or a rewriter's own throw answering `400`/`500` instead of forwarding).
+ *
+ * `mapErrorResponse` (task 2.14 fix round 1, findings-2.14-r1.md item 1; ADR
+ * `docs/decisions/2026-09-29-tensorrt-llm-sessions-carry-a-route-policy-to-the-public-server.md`) is the
+ * one exception on the response side, and only for errors: a non-2xx answer on a declared POST route
+ * is read (up to `MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES`) and answered with whatever OpenAI error the
+ * adapter maps it to, so a client talking to `SessionInfo.port` directly gets the same
+ * `context_length_exceeded` as one going through `:1337`. A 2xx answer is never read or parsed: it
+ * streams through as it arrives, exactly as before.
+ */
+import { createServer } from 'node:http';
+import { randomBytes } from 'node:crypto';
+import { StringDecoder } from 'node:string_decoder';
+import { AtomicCoreError } from '../../contracts/index.js';
+import { hostAndKeyGate } from '../../server/public/index.js';
+import { forwardableHeaders, pipeBody, readBody, relay, relayedHeaders, sendUpstream, sendWhole, } from '../../server/public/index.js';
+import { ManagedRequestRefusal } from './adapter.js';
+/** Hosts accepted as "loopback" for the upstream the gateway proxies to. */
+const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']);
+/** The gateway's own listener never binds anywhere else. */
+const GATEWAY_HOST = '127.0.0.1';
+/**
+ * `host`, bracketed when it is a bare IPv6 literal — what has to follow `http://` for `new URL` (and
+ * everything downstream of it) to parse the address instead of tripping over its colons. `::1` alone
+ * would otherwise be read as `host "::1"` with a bogus port after the last colon; `[::1]` already
+ * carries its own brackets and is left alone, as is every other accepted loopback spelling.
+ */
+export function bracketIfIpv6(host) {
+    return host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
+}
+/** Upstream connect timeout: generous because the container is local, but requests must not hang forever. */
+const CONNECT_TIMEOUT_MS = 30_000;
+/**
+ * The largest request body the rewrite path will read at all. Enforced *while the body streams in*
+ * (`readCappedBody`, below), not after it is fully buffered: reading stops the moment the running
+ * total crosses the cap, so an oversized body is never held in memory in full just to then be thrown
+ * away (findings-2.13-r2.md item 2). A declared route that is not rewritable never goes through this
+ * cap at all; it proxies through the same unbounded `readBody` every request used before any of this
+ * existed.
+ */
+export const MANAGED_GATEWAY_REWRITE_BODY_CAP_BYTES = 8 * 1024 * 1024;
+/**
+ * The largest engine *error* answer `mapErrorResponse` gets to read. An engine error is a short JSON
+ * document; one past this is answered with a generic error of the same status instead of being held
+ * in memory (findings-2.14-r1.md item 1: bounded, and only ever for non-2xx answers).
+ */
+export const MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES = 1024 * 1024;
+const OPENAI_ERROR_HEADERS = [['content-type', 'application/json']];
+/**
+ * An OpenAI-shaped `{"error": {message, type, code}}` body (findings-2.13-r2.md item 5) — the same
+ * envelope convention `server/public/errors.ts`'s `structureBackendErrorBody` already uses for
+ * llama.cpp/MLX, and `tensorrtLlmAdapter`'s own `context_length_exceeded` mapping, so a client never
+ * has to special-case a third error shape for the one gateway that happens to front a container.
+ */
+function sendOpenAIError(res, status, message, type, code) {
+    sendWhole(res, status, OPENAI_ERROR_HEADERS, JSON.stringify({ error: { message, type, code } }));
+}
+/**
+ * `405` for a path this session *does* declare, just not for this method (e.g. `GET
+ * /v1/chat/completions` when only `POST` is declared) — never forwarded, same as a `404`
+ * (findings-2.13-r3.md item 1). `Allow` lists every method actually declared for this path, per
+ * RFC 9110 §15.5.6. `HEAD` is never implied by a declared `GET`: an adapter that wants `HEAD`
+ * served declares it as its own route; the default (nothing declares it) lands here too, which is
+ * the safe answer for a route with no dedicated `HEAD` handler.
+ */
+function sendMethodNotAllowed(res, allowedMethods) {
+    const allow = allowedMethods.join(', ');
+    const body = JSON.stringify({
+        error: {
+            message: `Method not allowed on this route. Allowed: ${allow}.`,
+            type: 'invalid_request_error',
+            code: 'method_not_allowed',
+        },
+    });
+    sendWhole(res, 405, [...OPENAI_ERROR_HEADERS, ['allow', allow]], body);
+}
+/**
+ * Answers `413` for a body abandoned mid-read (over the rewrite cap) and only *then* — once the
+ * response has actually finished writing, via `res.end`'s own callback — destroys the connection.
+ * Order matters: destroying it before the response leaves would race the client into seeing a
+ * truncated response as a bare socket error instead of a clean `413` (this raced and failed exactly
+ * that way during development, when the destroy happened immediately instead of after the write).
+ * `req.destroy()` here is a deliberate, explicit call for a real `IncomingMessage`, even though
+ * `readCappedBody`'s own early `return` already tore down that same stream as a side effect of
+ * leaving its `for await` loop (see that function's own comment) — destroying an already-destroyed
+ * stream is a harmless no-op, and this call stays correct even for a hypothetical future `source`
+ * that is not a Node stream and so has no such side effect.
+ *
+ * Known, accepted limit (findings-2.13-r2.md item 2 / findings-2.13-r3.md item 5): a client that is
+ * still *actively writing* past the cap when the connection closes may never see this `413` body at
+ * all — it can observe a bare transport error instead (`EPIPE`/`ECONNRESET`), because closing a
+ * socket that still has unread bytes queued on it can make the OS send a reset instead of a clean
+ * close. This is not fixable from here (it is how TCP itself behaves, not a Node or gateway quirk),
+ * and is the same trade-off any server rejecting an oversized body mid-stream makes: protecting this
+ * process's memory against an oversized or runaway sender wins over guaranteeing that sender a
+ * pretty error message. A client that sends its body in one normal `write`-then-`end` (every real
+ * OpenAI client) sees the `413` cleanly, as `gateway.test.ts` covers.
+ */
+function sendTooLargeAndClose(req, res) {
+    const body = JSON.stringify({
+        error: {
+            message: 'Request body exceeds the maximum size accepted here.',
+            type: 'invalid_request_error',
+            code: 'request_too_large',
+        },
+    });
+    res.writeHead(413, [...OPENAI_ERROR_HEADERS, ['connection', 'close']].flat());
+    res.end(body, () => {
+        req.destroy();
+    });
+}
+/** The request path, with no query string — what a route is matched and rewritten on. */
+function routeOf(url) {
+    if (url === undefined)
+        return '/';
+    const qIndex = url.indexOf('?');
+    return qIndex === -1 ? url : url.slice(0, qIndex);
+}
+/**
+ * Percent-decodes a route path for matching against `options.routes`, refusing to do so at all when
+ * that would be ambiguous. An encoded slash (`%2f`/`%2F`) decoding into a literal `/` is the classic
+ * path-confusion trick — two request paths that look different before decoding could otherwise land
+ * on the same declared route, or a path that looks like it climbs into an unrelated route once
+ * decoded — so its mere presence makes the whole path unmatchable (`null`, which equals no entry in
+ * `routes`, so it always 404s) rather than trying to reason about which reading is "the real one".
+ * Malformed percent-encoding that `decodeURIComponent` itself rejects does the same.
+ */
+function decodedRoute(rawPath) {
+    if (/%2f/i.test(rawPath))
+        return null;
+    try {
+        return decodeURIComponent(rawPath);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Reads a body up to `capBytes` from any chunked async source (an `IncomingMessage` satisfies
+ * `AsyncIterable<Buffer>` on its own), counting while chunks arrive. The moment the running total
+ * would exceed the cap, reading stops immediately — the source is never asked for another chunk,
+ * and nothing past the cap is ever buffered — and `'too-large'` comes back instead of a `Buffer`
+ * (findings-2.13-r2.md item 2: exported, and typed against the generic interface rather than
+ * `IncomingMessage` specifically, so this exact claim — stops pulling before the cap, not after —
+ * has a direct unit test against a source that never ends, instead of only an HTTP-level test whose
+ * client would have to keep sending past a connection the server has already closed, which is an
+ * inherently racy thing to assert on at the TCP level).
+ *
+ * That early `return` is not as passive as it looks (findings-2.13-r3.md item 4): exiting a `for
+ * await...of` loop before it runs to completion — by `return`, `break` or an uncaught throw — makes
+ * the language call the async iterator's own `.return()`, and for a Node.js `Readable` (which is
+ * what a real `IncomingMessage` is), that already tears the stream down as a side effect, before
+ * this function's caller ever gets a chance to decide anything. So for the real gateway path, the
+ * connection's read side is already gone by the time `readCappedBody` returns `'too-large'` — this
+ * function does not *decide* to leave the connection alone, the language leaves it no choice in the
+ * matter. (This has no such effect on a `source` that is not backed by a Node stream — an async
+ * generator, say, as this file's own unit test for this function uses — where "stop iterating" truly
+ * is just that.) `sendTooLargeAndClose` still calls `req.destroy()` itself explicitly, both because
+ * destroying an already-destroyed stream is a harmless no-op and because relying on this implicit
+ * side effect would silently stop working for a source that is not a plain Node stream.
+ */
+export async function readCappedBody(source, capBytes) {
+    const chunks = [];
+    let total = 0;
+    for await (const chunk of source) {
+        total += chunk.length;
+        if (total > capBytes)
+            return 'too-large';
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+}
+/**
+ * A fresh, unguessable session key: 256 bits of `crypto.randomBytes`, base64url so it fits a Bearer
+ * header verbatim. Call this once per generation — a reload gets a new key, so a caller holding the
+ * previous one is refused (`SESSION_GENERATION_STALE` at the session layer; here, plain 401).
+ */
+export function generateGatewayKey() {
+    return randomBytes(32).toString('base64url');
+}
+/**
+ * Sends `body` (already fully read) to the upstream and relays its answer back unmodified,
+ * streamed or not. Shared by the rewritten and the byte-for-byte paths — everything from here down
+ * has never cared, and still does not care, which one a request came through.
+ */
+async function sendBodyToUpstream(req, res, upstream, body, mapError, responseRewrite = null, streamFailure) {
+    // A client that disconnects before the upstream has even answered must not leave that connect
+    // attempt, or a slow-to-respond container, running unattended.
+    const controller = new AbortController();
+    const onEarlyClose = () => controller.abort();
+    res.once('close', onEarlyClose);
+    let upstreamResponse;
+    try {
+        const origin = `http://${bracketIfIpv6(upstream.host)}:${upstream.port}`;
+        upstreamResponse = await sendUpstream(`${origin}${req.url ?? '/'}`, {
+            method: req.method ?? 'GET',
+            // The client's own auth headers are this gateway's secret, not the container's; they stop here.
+            headers: forwardableHeaders(req, ['authorization', 'x-api-key']),
+            ...(body.length > 0 ? { body } : {}),
+            connectTimeoutMs: CONNECT_TIMEOUT_MS,
+            signal: controller.signal,
+        });
+    }
+    catch {
+        res.off('close', onEarlyClose);
+        if (!res.headersSent && !res.destroyed)
+            sendWhole(res, 502, [], 'Bad Gateway');
+        return;
+    }
+    res.off('close', onEarlyClose);
+    const status = upstreamResponse.status;
+    if (mapError !== undefined && (status < 200 || status > 299)) {
+        await answerEngineError(res, upstreamResponse, mapError);
+        return;
+    }
+    if (status >= 200 && status <= 299 && isEventStream(upstreamResponse)) {
+        res.writeHead(upstreamResponse.status, relayedHeaders(upstreamResponse, []).flat());
+        await pipeBody(res, guardedEventStream(upstreamResponse.body, responseRewrite, streamFailure), () => upstreamResponse.body.destroy());
+        return;
+    }
+    if (responseRewrite !== null && status >= 200 && status <= 299) {
+        await relayRewritten(res, upstreamResponse, responseRewrite);
+        return;
+    }
+    // `relay` owns the rest: it writes the status/headers, pipes the body with backpressure, and
+    // tears the upstream connection down itself if the client disconnects mid-stream.
+    await relay(res, upstreamResponse, []);
+}
+/** A JSON text rewritten by `rewrite`, or the text itself when it is not a JSON object. */
+function rewriteJsonText(text, rewrite) {
+    try {
+        const parsed = JSON.parse(text);
+        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed))
+            return text;
+        return JSON.stringify(rewrite(parsed));
+    }
+    catch {
+        return text;
+    }
+}
+function isEventStream(upstream) {
+    const type = upstream.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? '';
+    return type.toLowerCase().includes('text/event-stream');
+}
+/** What a client reads when the engine ends a stream without finishing it and says nothing better. */
+export const MANAGED_STREAM_CUT_MESSAGE = 'The engine stopped the answer before finishing it.';
+/**
+ * Each complete SSE line of a 2xx stream, `data:` JSON events through `rewrite` when there is one,
+ * everything else verbatim — and an end the client can tell from a finished answer: a stream that
+ * breaks off or closes without `data: [DONE]` gets an OpenAI error event (`streamFailure`'s words,
+ * else {@link MANAGED_STREAM_CUT_MESSAGE}) and then `[DONE]`. Without it an engine that failed a
+ * request after `200 OK` (TensorRT-LLM's `RequestError` when the KV cache cannot hold it) reached the
+ * client as a clean, empty answer: `pipeBody` swallows an upstream failure mid-stream.
+ */
+async function* guardedEventStream(source, rewrite, streamFailure) {
+    // A multi-byte character can straddle two chunks; the decoder holds its first bytes back.
+    const decoder = new StringDecoder('utf8');
+    let pending = '';
+    let done = false;
+    const line = (raw) => {
+        const match = /^data:\s?(.*)$/.exec(raw.replace(/\r$/, ''));
+        if (match === null)
+            return raw;
+        const payload = match[1].trim();
+        if (payload === '[DONE]')
+            done = true;
+        if (payload === '' || payload === '[DONE]' || rewrite === null)
+            return raw;
+        return `data: ${rewriteJsonText(payload, rewrite)}`;
+    };
+    try {
+        for await (const chunk of source) {
+            pending += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+            const lines = pending.split('\n');
+            pending = lines.pop();
+            if (lines.length > 0)
+                yield `${lines.map(line).join('\n')}\n`;
+        }
+    }
+    catch {
+        // The upstream broke off mid-stream; what follows tells the client so.
+    }
+    pending += decoder.end();
+    if (pending !== '')
+        yield `${line(pending)}\n`;
+    if (done)
+        return;
+    const message = (await streamFailure?.().catch(() => null)) ?? MANAGED_STREAM_CUT_MESSAGE;
+    const error = { error: { message, type: 'server_error', code: 'stream_interrupted' } };
+    yield `\ndata: ${JSON.stringify(error)}\n\ndata: [DONE]\n\n`;
+}
+/** A 2xx JSON answer through `rewrite`, as a whole (event streams go through `guardedEventStream`). */
+async function relayRewritten(res, upstream, rewrite) {
+    const whole = await readCappedBody(upstream.body, MANAGED_GATEWAY_REWRITE_BODY_CAP_BYTES * 64).catch(() => null);
+    if (whole === null || whole === 'too-large') {
+        if (!res.headersSent && !res.destroyed)
+            sendWhole(res, 502, [], 'Bad Gateway');
+        return;
+    }
+    sendWhole(res, upstream.status, relayedHeaders(upstream, []), rewriteJsonText(whole.toString('utf8'), rewrite));
+}
+/**
+ * An engine's error answer, read under `MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES`: the adapter's mapped
+ * OpenAI error when it has one, otherwise the engine's own answer (status, headers, bytes) as it was.
+ * A client that leaves while the body is still arriving tears the engine connection down with it
+ * (final review M-4): an engine stalled mid-error would otherwise hold that socket forever. An
+ * answer that cannot be relayed keeps the engine's status, labelled a client error for a `4xx`.
+ */
+async function answerEngineError(res, upstream, mapError) {
+    const onClientGone = () => upstream.body.destroy();
+    res.once('close', onClientGone);
+    let capped;
+    try {
+        capped = res.destroyed
+            ? null
+            : await readCappedBody(upstream.body, MANAGED_GATEWAY_ERROR_BODY_CAP_BYTES).catch(() => null);
+    }
+    finally {
+        res.off('close', onClientGone);
+    }
+    if (res.destroyed) {
+        upstream.body.destroy();
+        return;
+    }
+    if (capped === null || capped === 'too-large') {
+        upstream.body.destroy();
+        const clientError = upstream.status >= 400 && upstream.status < 500;
+        sendOpenAIError(res, upstream.status, 'The engine answered with an error that could not be relayed.', clientError ? 'invalid_request_error' : 'server_error', 'upstream_error');
+        return;
+    }
+    const mapped = mapError(upstream.status, capped.toString('utf8'));
+    if (mapped !== null) {
+        sendWhole(res, upstream.status, OPENAI_ERROR_HEADERS, JSON.stringify(mapped));
+        return;
+    }
+    sendWhole(res, upstream.status, relayedHeaders(upstream, []), capped);
+}
+/**
+ * Reads the client's body under the rewrite cap, parses it as JSON, hands it to `rewriteRequestBody`
+ * and re-serializes the result — or answers `413`/`400`/`500` itself and returns `null`, meaning the
+ * caller must not proceed to the upstream at all. `sendUpstream` derives `Content-Length` from
+ * whatever buffer it is actually given, so a body that grew or shrank under rewriting is never sent
+ * with the client's original, now-stale length (and never as `Transfer-Encoding: chunked`, whatever
+ * the client itself sent it as — the whole point of reading it here first).
+ */
+async function readAndRewriteBody(req, res, route, rewriteRequestBody) {
+    const capped = await readCappedBody(req, MANAGED_GATEWAY_REWRITE_BODY_CAP_BYTES);
+    if (capped === 'too-large') {
+        sendTooLargeAndClose(req, res);
+        return null;
+    }
+    if (capped.length === 0)
+        return capped;
+    let parsed;
+    try {
+        parsed = JSON.parse(capped.toString('utf8'));
+    }
+    catch {
+        sendOpenAIError(res, 400, 'Request body is not valid JSON.', 'invalid_request_error', 'invalid_json');
+        return null;
+    }
+    let rewritten;
+    try {
+        rewritten = rewriteRequestBody(route, parsed);
+    }
+    catch (error) {
+        // Only a validation error — the adapter rejecting something the *client* sent — is safe to
+        // surface verbatim as a 400: its message was written to be read by that client. Anything else
+        // (a bug in the adapter's own rewrite logic, for instance) becomes a generic 500 instead of
+        // leaking an internal error message to the client; the detail still goes to this process's own
+        // log, since this module has no injected logger of its own to hand it to (findings-2.13-r3.md
+        // item 3).
+        if (error instanceof ManagedRequestRefusal) {
+            sendOpenAIError(res, 400, error.message, 'invalid_request_error', error.openaiCode);
+        }
+        else if (error instanceof AtomicCoreError && error.code === 'INVALID_ARGUMENT') {
+            sendOpenAIError(res, 400, error.message, 'invalid_request_error', 'invalid_request_error');
+        }
+        else {
+            console.error('managed-text gateway: rewriteRequestBody threw an unexpected error:', error);
+            sendOpenAIError(res, 500, 'The request could not be processed.', 'server_error', 'internal_error');
+        }
+        return null;
+    }
+    return Buffer.from(JSON.stringify(rewritten), 'utf8');
+}
+/**
+ * Matches the request against `options.routes`, method and path together (findings-2.13-r3.md item
+ * 1): no declared route for this path at all is `404`; the path is declared, just not for this
+ * method, is `405` with an `Allow` header. Neither is ever forwarded, and neither reads the body.
+ * Otherwise, either rewrites the body (a declared, rewritable route, `rewriteRequestBody`
+ * configured) or proxies it through untouched — the same byte-for-byte path every request took
+ * before rewriting existed at all, still used for every other route.
+ */
+async function proxyToUpstream(req, res, options) {
+    const path = decodedRoute(routeOf(req.url));
+    const method = req.method ?? 'GET';
+    const declaredForPath = path === null ? [] : options.routes.filter((r) => r.path === path);
+    if (path === null || declaredForPath.length === 0) {
+        sendOpenAIError(res, 404, 'No route on this session matches this request.', 'invalid_request_error', 'not_found');
+        return;
+    }
+    if (!declaredForPath.some((r) => r.method === method)) {
+        sendMethodNotAllowed(res, declaredForPath.map((r) => r.method));
+        return;
+    }
+    const route = path;
+    const rewriteRequestBody = options.rewriteRequestBody;
+    const rewritable = rewriteRequestBody !== undefined &&
+        (options.rewritableRoutes ?? []).some((r) => r.method === method && r.path === route);
+    let body;
+    if (rewritable) {
+        const rewritten = await readAndRewriteBody(req, res, route, rewriteRequestBody);
+        if (rewritten === null)
+            return; // readAndRewriteBody already answered the client.
+        body = rewritten;
+    }
+    else {
+        try {
+            body = await readBody(req);
+        }
+        catch {
+            if (!res.headersSent)
+                sendOpenAIError(res, 400, 'Failed to read the request body.', 'invalid_request_error', 'invalid_request_error');
+            return;
+        }
+    }
+    // Only a declared POST route's error answer is ever read: that is where an engine reports what a
+    // client sent it (a prompt past the context), and where a client expects an OpenAI error back.
+    const mapErrorResponse = options.mapErrorResponse;
+    const mapError = mapErrorResponse !== undefined && method === 'POST'
+        ? (status, text) => mapErrorResponse(route, status, text)
+        : undefined;
+    // A response rewrite is chosen from the request the engine actually gets (rewritable routes only).
+    let responseRewrite = null;
+    if (rewritable && options.rewriteResponseFor !== undefined && body.length > 0) {
+        try {
+            responseRewrite = options.rewriteResponseFor(route, JSON.parse(body.toString('utf8')));
+        }
+        catch {
+            responseRewrite = null;
+        }
+    }
+    await sendBodyToUpstream(req, res, options.upstream, body, mapError, responseRewrite, options.streamFailure);
+}
+function assertLoopback(host) {
+    if (!LOOPBACK_HOSTS.has(host.toLowerCase())) {
+        throw new AtomicCoreError('INVALID_ARGUMENT', 'Managed session gateway upstream must be loopback.', `got host "${host}"`);
+    }
+}
+/**
+ * `trtllm-serve` checks no key of its own — this gateway is the *only* thing standing between the
+ * container's port and anything on the machine (or, via DNS rebinding, a web page). `hostAndKeyGate`
+ * treats an empty `apiKey` as "auth disabled", which is the right default for `:1337` (a user opted
+ * out in Settings) but is never correct here: nobody can opt a managed session out of its own only
+ * guard. A caller starting one without a key is a bug, not a configuration choice, so it fails loud.
+ */
+function assertApiKeyPresent(apiKey) {
+    if (apiKey === '') {
+        throw new AtomicCoreError('INVALID_ARGUMENT', 'Managed session gateway requires a non-empty api key: it is the only auth in front of an ' +
+            'engine that checks none of its own.');
+    }
+}
+/**
+ * Start a gateway for one managed session: binds `127.0.0.1:0`, gates every request on the Host and
+ * Bearer checks, and streams whatever passes through to `options.upstream`.
+ */
+export async function startManagedGateway(options) {
+    assertLoopback(options.upstream.host);
+    assertApiKeyPresent(options.apiKey);
+    const gateConfig = { apiKey: options.apiKey, trustedHosts: options.allowedHosts };
+    const server = createServer({ requireHostHeader: false }, (req, res) => {
+        const refused = hostAndKeyGate(req, gateConfig);
+        if (refused) {
+            sendWhole(res, refused.status, [], refused.body);
+            return;
+        }
+        proxyToUpstream(req, res, options).catch(() => {
+            if (!res.headersSent && !res.destroyed)
+                sendWhole(res, 502, [], 'Bad Gateway');
+            else
+                res.destroy();
+        });
+    });
+    await new Promise((resolve, reject) => {
+        server.once('error', reject);
+        server.listen(0, GATEWAY_HOST, () => resolve());
+    });
+    const address = server.address();
+    const port = typeof address === 'object' && address !== null ? address.port : 0;
+    return {
+        host: GATEWAY_HOST,
+        port,
+        close: () => new Promise((resolve) => {
+            server.closeAllConnections?.();
+            server.close(() => resolve());
+        }),
+    };
+}
+//# sourceMappingURL=gateway.js.map

package/dist/runtime/managed-text/gateway.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway.js","sourceRoot":"","sources":["../../../src/runtime/managed-text/gateway.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAE7D,OAAO,EACL,kBAAkB,EAClB,QAAQ,EACR,QAAQ,EACR,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,GACV,MAAM,8BAA8B,CAAA;AAErC,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AAGpD,4EAA4E;AAC5E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;AAE1E,4DAA4D;AAC5D,MAAM,YAAY,GAAG,WAAW,CAAA;AAEhC;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;AACzE,CAAC;AAyDD,6GAA6G;AAC7G,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAErE;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,IAAI,GAAG,IAAI,CAAA;AAE/D,MAAM,oBAAoB,GAAgB,CAAC,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAA;AAEhF;;;;;GAKG;AACH,SAAS,eAAe,CACtB,GAAmB,EACnB,MAAc,EACd,OAAe,EACf,IAAY,EACZ,IAAY;IAEZ,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAA;AAClG,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,oBAAoB,CAAC,GAAmB,EAAE,cAAiC;IAClF,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1B,KAAK,EAAE;YACL,OAAO,EAAE,8CAA8C,KAAK,GAAG;YAC/D,IAAI,EAAE,uBAAuB;YAC7B,IAAI,EAAE,oBAAoB;SAC3B;KACF,CAAC,CAAA;IACF,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,oBAAoB,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;AACxE,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,oBAAoB,CAAC,GAAoB,EAAE,GAAmB;IACrE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1B,KAAK,EAAE;YACL,OAAO,EAAE,sDAAsD;YAC/D,IAAI,EAAE,uBAAuB;YAC7B,IAAI,EAAE,mBAAmB;SAC1B;KACF,CAAC,CAAA;IACF,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,oBAAoB,EAAE,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;IAC7E,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE;QACjB,GAAG,CAAC,OAAO,EAAE,CAAA;IACf,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,yFAAyF;AACzF,SAAS,OAAO,CAAC,GAAuB;IACtC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,GAAG,CAAA;IACjC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC/B,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;AACnD,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,OAAe;IACnC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IACrC,IAAI,CAAC;QACH,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAA;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAA6B,EAC7B,QAAgB;IAEhB,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAA;QACrB,IAAI,KAAK,GAAG,QAAQ;YAAE,OAAO,WAAW,CAAA;QACxC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACpB,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AAC9B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,kBAAkB,CAC/B,GAAoB,EACpB,GAAmB,EACnB,QAAgC,EAChC,IAAY,EACZ,QAA0D,EAC1D,kBAAiD,IAAI,EACrD,aAA4C;IAE5C,8FAA8F;IAC9F,+DAA+D;IAC/D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;IAC7C,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAE/B,IAAI,gBAAkC,CAAA;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAA;QACxE,gBAAgB,GAAG,MAAM,YAAY,CAAC,GAAG,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,EAAE;YAClE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,KAAK;YAC3B,gGAAgG;YAChG,OAAO,EAAE,kBAAkB,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,WAAW,CAAC,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,gBAAgB,EAAE,kBAAkB;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAA;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;QAC9B,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,SAAS;YAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,aAAa,CAAC,CAAA;QAC9E,OAAM;IACR,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC9B,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAA;IACtC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;QAC7D,MAAM,iBAAiB,CAAC,GAAG,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAAA;QACxD,OAAM;IACR,CAAC;IACD,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAI,aAAa,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACtE,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACnF,MAAM,QAAQ,CAAC,GAAG,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,EAAE,eAAe,EAAE,aAAa,CAAC,EAAE,GAAG,EAAE,CAClG,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,CAChC,CAAA;QACD,OAAM;IACR,CAAC;IACD,IAAI,eAAe,KAAK,IAAI,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAC/D,MAAM,cAAc,CAAC,GAAG,EAAE,gBAAgB,EAAE,eAAe,CAAC,CAAA;QAC5D,OAAM;IACR,CAAC;IACD,6FAA6F;IAC7F,kFAAkF;IAClF,MAAM,KAAK,CAAC,GAAG,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAA;AACxC,CAAC;AAED,2FAA2F;AAC3F,SAAS,eAAe,CAAC,IAAY,EAAE,OAA+B;IACpE,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAA;QACvF,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAiC,CAAC,CAAC,CAAA;IACnE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,QAA0B;IAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAChG,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;AACzD,CAAC;AAED,sGAAsG;AACtG,MAAM,CAAC,MAAM,0BAA0B,GAAG,oDAAoD,CAAA;AAE9F;;;;;;;GAOG;AACH,KAAK,SAAS,CAAC,CAAC,kBAAkB,CAChC,MAAsC,EACtC,OAAsC,EACtC,aAA4C;IAE5C,0FAA0F;IAC1F,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,IAAI,GAAG,CAAC,GAAW,EAAU,EAAE;QACnC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAA;QAC3D,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,GAAG,CAAA;QAC9B,MAAM,OAAO,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,IAAI,EAAE,CAAA;QAC3C,IAAI,OAAO,KAAK,QAAQ;YAAE,IAAI,GAAG,IAAI,CAAA;QACrC,IAAI,OAAO,KAAK,EAAE,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO,GAAG,CAAA;QAC1E,OAAO,SAAS,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAA;IACrD,CAAC,CAAA;IACD,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,OAAO,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACjC,OAAO,GAAG,KAAK,CAAC,GAAG,EAAY,CAAA;YAC/B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAA;QAC/D,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;IACzE,CAAC;IACD,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACxB,IAAI,OAAO,KAAK,EAAE;QAAE,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAA;IAC9C,IAAI,IAAI;QAAE,OAAM;IAChB,MAAM,OAAO,GAAG,CAAC,MAAM,aAAa,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,0BAA0B,CAAA;IACzF,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,oBAAoB,EAAE,EAAE,CAAA;IACtF,MAAM,WAAW,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,sBAAsB,CAAA;AAC9D,CAAC;AAED,uGAAuG;AACvG,KAAK,UAAU,cAAc,CAC3B,GAAmB,EACnB,QAA0B,EAC1B,OAA+B;IAE/B,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,sCAAsC,GAAG,EAAE,CAAC,CAAC,KAAK,CAClG,GAAG,EAAE,CAAC,IAAI,CACX,CAAA;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,WAAW,EAAE,CAAC;QAC5C,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,SAAS;YAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,aAAa,CAAC,CAAA;QAC9E,OAAM;IACR,CAAC;IACD,SAAS,CACP,GAAG,EACH,QAAQ,CAAC,MAAM,EACf,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,EAC5B,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CACjD,CAAA;AACH,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,iBAAiB,CAC9B,GAAmB,EACnB,QAA0B,EAC1B,QAAyD;IAEzD,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA;IAClD,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC/B,IAAI,MAAmC,CAAA;IACvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,SAAS;YACpB,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,MAAM,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IACjG,CAAC;YAAS,CAAC;QACT,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAChC,CAAC;IACD,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA;QACvB,OAAM;IACR,CAAC;IACD,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA;QACvB,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAA;QACnE,eAAe,CACb,GAAG,EACH,QAAQ,CAAC,MAAM,EACf,8DAA8D,EAC9D,WAAW,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,cAAc,EACtD,gBAAgB,CACjB,CAAA;QACD,OAAM;IACR,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IACjE,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,SAAS,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;QAC7E,OAAM;IACR,CAAC;IACD,SAAS,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;AACvE,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,kBAAkB,CAC/B,GAAoB,EACpB,GAAmB,EACnB,KAAa,EACb,kBAA6D;IAE7D,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,sCAAsC,CAAC,CAAA;IAChF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC9B,OAAO,IAAI,CAAA;IACb,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAA;IAEtC,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,iCAAiC,EAAE,uBAAuB,EAAE,cAAc,CAAC,CAAA;QACrG,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,SAAkB,CAAA;IACtB,IAAI,CAAC;QACH,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2FAA2F;QAC3F,8FAA8F;QAC9F,4FAA4F;QAC5F,+FAA+F;QAC/F,8FAA8F;QAC9F,WAAW;QACX,IAAI,KAAK,YAAY,qBAAqB,EAAE,CAAC;YAC3C,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,uBAAuB,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;QACrF,CAAC;aAAM,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;YACjF,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,CAAC,CAAA;QAC5F,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,qEAAqE,EAAE,KAAK,CAAC,CAAA;YAC3F,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,qCAAqC,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAAA;QACpG,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAA;AACvD,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAoB,EACpB,GAAmB,EACnB,OAA8B;IAE9B,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,CAAA;IAClC,MAAM,eAAe,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;IAC1F,IAAI,IAAI,KAAK,IAAI,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,eAAe,CACb,GAAG,EACH,GAAG,EACH,gDAAgD,EAChD,uBAAuB,EACvB,WAAW,CACZ,CAAA;QACD,OAAM;IACR,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;QACtD,oBAAoB,CAClB,GAAG,EACH,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CACrC,CAAA;QACD,OAAM;IACR,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAA;IAElB,MAAM,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAA;IACrD,MAAM,UAAU,GACd,kBAAkB,KAAK,SAAS;QAChC,CAAC,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAA;IAEvF,IAAI,IAAY,CAAA;IAChB,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAA;QAC/E,IAAI,SAAS,KAAK,IAAI;YAAE,OAAM,CAAC,kDAAkD;QACjF,IAAI,GAAG,SAAS,CAAA;IAClB,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,GAAG,CAAC,WAAW;gBAClB,eAAe,CACb,GAAG,EACH,GAAG,EACH,kCAAkC,EAClC,uBAAuB,EACvB,uBAAuB,CACxB,CAAA;YACH,OAAM;QACR,CAAC;IACH,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAA;IACjD,MAAM,QAAQ,GACZ,gBAAgB,KAAK,SAAS,IAAI,MAAM,KAAK,MAAM;QACjD,CAAC,CAAC,CAAC,MAAc,EAAE,IAAY,EAAE,EAAE,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC;QACzE,CAAC,CAAC,SAAS,CAAA;IACf,mGAAmG;IACnG,IAAI,eAAe,GAAkC,IAAI,CAAA;IACzD,IAAI,UAAU,IAAI,OAAO,CAAC,kBAAkB,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9E,IAAI,CAAC;YACH,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACxF,CAAC;QAAC,MAAM,CAAC;YACP,eAAe,GAAG,IAAI,CAAA;QACxB,CAAC;IACH,CAAC;IACD,MAAM,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;AAC9G,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,eAAe,CACvB,kBAAkB,EAClB,oDAAoD,EACpD,aAAa,IAAI,GAAG,CACrB,CAAA;IACH,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,MAAc;IACzC,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAClB,MAAM,IAAI,eAAe,CACvB,kBAAkB,EAClB,2FAA2F;YACzF,qCAAqC,CACxC,CAAA;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAA8B;IACtE,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,UAAU,GAAqB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAA;IAEnG,MAAM,MAAM,GAAW,YAAY,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC7E,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;QAC/C,IAAI,OAAO,EAAE,CAAC;YACZ,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;YAChD,OAAM;QACR,CAAC;QACD,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAC5C,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,SAAS;gBAAE,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,aAAa,CAAC,CAAA;;gBACzE,GAAG,CAAC,OAAO,EAAE,CAAA;QACpB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAC5B,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,IAAI;QACJ,KAAK,EAAE,GAAG,EAAE,CACV,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5B,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAA;YAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAC/B,CAAC,CAAC;KACL,CAAA;AACH,CAAC"}

package/dist/runtime/managed-text/generation-defaults.d.ts

@@ -0,0 +1,23 @@
+/** In the names both engines' sampling parameters use. */
+export interface GenerationDefaults {
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    min_p?: number;
+    repetition_penalty?: number;
+}
+/** The stored key of each default, by its sampling-parameter name. */
+export declare const GENERATION_DEFAULT_KEYS: {
+    readonly temperature: "default_temperature";
+    readonly top_p: "default_top_p";
+    readonly top_k: "default_top_k";
+    readonly min_p: "default_min_p";
+    readonly repetition_penalty: "default_repetition_penalty";
+};
+/**
+ * The defaults `raw` sets, validated; `INVALID_ARGUMENT` for one out of range. `raw` holds the stored
+ * keys (`default_temperature`, …); a value may be a number or its text, `''`/`null` is not set.
+ */
+export declare function readGenerationDefaults(provider: string, raw: Record<string, unknown>): GenerationDefaults;
+/** `body` with every default it does not set itself. */
+export declare function withGenerationDefaults(body: Record<string, unknown>, defaults: GenerationDefaults): Record<string, unknown>;

package/dist/runtime/managed-text/generation-defaults.js

@@ -0,0 +1,51 @@
+/**
+ * Sampling defaults a managed provider applies to a request that sets none (change
+ * `add-vllm-runtime`, owner's decision after the live run): the settings `default_temperature`,
+ * `default_top_p`, `default_top_k`, `default_min_p` and `default_repetition_penalty`, read once for
+ * every engine. vLLM receives them in `--override-generation-config`; TensorRT-LLM, which has no such
+ * launch option, gets them from the session gateway, written into a request that names none. `''`
+ * (or nothing stored) is "not set" — the model's own default stands; a request's own value always wins.
+ */
+import { AtomicCoreError } from '../../contracts/index.js';
+/** The stored key of each default, by its sampling-parameter name. */
+export const GENERATION_DEFAULT_KEYS = {
+    temperature: 'default_temperature',
+    top_p: 'default_top_p',
+    top_k: 'default_top_k',
+    min_p: 'default_min_p',
+    repetition_penalty: 'default_repetition_penalty',
+};
+/** The bounds both engines accept (vLLM's `SamplingParams._verify_args`; TRT-LLM's are looser). */
+const RULES = {
+    temperature: { valid: (n) => n >= 0 && n <= 2, range: 'a number from 0 to 2' },
+    top_p: { valid: (n) => n > 0 && n <= 1, range: 'a number above 0 and up to 1' },
+    top_k: { valid: (n) => Number.isInteger(n) && n >= -1 && n <= 1_000_000, range: 'a whole number from -1' },
+    min_p: { valid: (n) => n >= 0 && n <= 1, range: 'a number from 0 to 1' },
+    repetition_penalty: { valid: (n) => n > 0 && n <= 10, range: 'a number above 0 and up to 10' },
+};
+/**
+ * The defaults `raw` sets, validated; `INVALID_ARGUMENT` for one out of range. `raw` holds the stored
+ * keys (`default_temperature`, …); a value may be a number or its text, `''`/`null` is not set.
+ */
+export function readGenerationDefaults(provider, raw) {
+    const defaults = {};
+    for (const [name, key] of Object.entries(GENERATION_DEFAULT_KEYS)) {
+        const value = raw[key];
+        if (value === undefined || value === null)
+            continue;
+        if (typeof value === 'string' && value.trim() === '')
+            continue;
+        const number = typeof value === 'string' ? Number(value) : value;
+        if (typeof number !== 'number' || !Number.isFinite(number) || !RULES[name].valid(number)) {
+            throw new AtomicCoreError('INVALID_ARGUMENT', `${provider} setting ${key} must be ${RULES[name].range}.`, `${key}=${JSON.stringify(value)}`);
+        }
+        defaults[name] = number;
+    }
+    return defaults;
+}
+/** `body` with every default it does not set itself. */
+export function withGenerationDefaults(body, defaults) {
+    const missing = Object.entries(defaults).filter(([name]) => body[name] === undefined || body[name] === null);
+    return missing.length === 0 ? body : { ...body, ...Object.fromEntries(missing) };
+}
+//# sourceMappingURL=generation-defaults.js.map

package/dist/runtime/managed-text/generation-defaults.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generation-defaults.js","sourceRoot":"","sources":["../../../src/runtime/managed-text/generation-defaults.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAW1D,sEAAsE;AACtE,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,WAAW,EAAE,qBAAqB;IAClC,KAAK,EAAE,eAAe;IACtB,KAAK,EAAE,eAAe;IACtB,KAAK,EAAE,eAAe;IACtB,kBAAkB,EAAE,4BAA4B;CACxC,CAAA;AAIV,mGAAmG;AACnG,MAAM,KAAK,GAAmE;IAC5E,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE;IAC9E,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE;IAC/E,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,KAAK,EAAE,wBAAwB,EAAE;IAC1G,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,sBAAsB,EAAE;IACxE,kBAAkB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,+BAA+B,EAAE;CAC/F,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,GAA4B;IACnF,MAAM,QAAQ,GAAuB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAqB,EAAE,CAAC;QACtF,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,SAAQ;QACnD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAQ;QAC9D,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;QAChE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACzF,MAAM,IAAI,eAAe,CACvB,kBAAkB,EAClB,GAAG,QAAQ,YAAY,GAAG,YAAY,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,EAC1D,GAAG,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAClC,CAAA;QACH,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAA;IACzB,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,sBAAsB,CACpC,IAA6B,EAC7B,QAA4B;IAE5B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAA;IAC5G,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAA;AAClF,CAAC"}