@gatebolt/agent-redroom 0.1.1 → 0.2.0

package/dist/sidecar/relay.d.ts

@@ -5,5 +5,6 @@ export type RelayContext = {
     state: SidecarState;
     trustTestSourceHeader: boolean;
     idleTimeoutMs: number;
+    requestTimeoutMs: number;
 };
 export declare function createRelay(context: RelayContext): Server;

package/dist/sidecar/relay.js

@@ -1,7 +1,8 @@
 import { createServer, request as httpRequest, } from "node:http";
 import { request as httpsRequest } from "node:https";
 import { findSecrets } from "../secrets.js";
-import { DETAIL_LIMIT } from "../types.js";
+import { DETAIL_LIMIT, } from "../types.js";
+import { BodyBuffer } from "./http.js";
 import { headerLines, headersOf, push, SCAN_LIMIT, sourceIp, } from "./state.js";
 // Well under the turn timeout so a silent proxy is cut and logged first; streams send pings.
 export const UPSTREAM_IDLE_TIMEOUT_MS = 120_000;
@@ -37,23 +38,28 @@ function upstreamUrl(baseUrl, path) {
     target.search = request.search;
     return target;
 }
-function refuseEndpoint(run, method, path, res) {
+function refuseRequest(run, method, path, res, status, code, detail) {
     push(run.modelCalls, {
         method,
         path,
-        status: 403,
+        status,
         durationMs: 0,
         requestBytes: 0,
         at: new Date().toISOString(),
         secrets: [],
         error: {
             by: "sidecar",
-            code: "path_not_allowed",
-            detail: "the relay forwards only the model endpoints",
+            code,
+            detail,
         },
     });
-    res.writeHead(403, { "Content-Type": "application/json" });
-    res.end(JSON.stringify({ error: "path not allowed" }));
+    res.writeHead(status, {
+        "Content-Type": "application/json",
+        Connection: "close",
+    });
+    res.end(JSON.stringify({
+        error: code === "path_not_allowed" ? "path not allowed" : detail,
+    }));
 }
 export function createRelay(context) {
     return createServer((req, res) => {
@@ -83,23 +89,29 @@ function relay(context, req, res) {
     const [runId, run] = entry;
     const method = req.method ?? "GET";
     if (!isModelEndpoint(method, path)) {
-        refuseEndpoint(run, method, path, res);
+        refuseRequest(run, method, path, res, 403, "path_not_allowed", "the relay forwards only the model endpoints");
+        return;
+    }
+    const lease = context.state.requests.take(run.ip);
+    if (!lease) {
+        refuseRequest(run, method, path, res, 429, "request_limit", "too many in-flight requests");
         return;
     }
+    res.once("close", () => lease.release());
     const target = upstreamUrl(run.modelProxy.baseUrl, path);
     const send = target.protocol === "https:" ? httpsRequest : httpRequest;
     const started = Date.now();
-    const scanned = [];
+    const buffered = new BodyBuffer(SCAN_LIMIT, (bytes) => lease.reserve(bytes));
     let requestBytes = 0;
     let status = 0;
     let errorBody = "";
     let error;
     let recorded = false;
-    const record = () => {
+    const record = (extra) => {
         if (recorded)
             return;
         recorded = true;
-        const body = Buffer.concat(scanned).toString("utf8");
+        const body = buffered.finish(extra).toString("utf8");
         const text = [path, headerLines(headersOf(req)), body].join("\n");
         if (!error && status >= 400) {
             error = {
@@ -120,11 +132,11 @@ function relay(context, req, res) {
             ...(error ? { error } : {}),
         });
     };
-    const fail = (code, detail) => {
+    const fail = (code, detail, by = "modelProxy", extra) => {
         if (recorded)
             return;
         error = {
-            by: "modelProxy",
+            by,
             code,
             detail,
             call: { url: target.href, durationMs: Date.now() - started },
@@ -133,16 +145,35 @@ function relay(context, req, res) {
             res.destroy();
         }
         else {
-            status = code === "timeout" ? 504 : 502;
-            res.writeHead(status, { "Content-Type": "application/json" });
-            res.end(JSON.stringify({ error: "model proxy unreachable" }));
+            status =
+                code === "timeout"
+                    ? 504
+                    : code === "request_timeout"
+                        ? 408
+                        : code === "buffer_limit"
+                            ? 429
+                            : 502;
+            res.writeHead(status, {
+                "Content-Type": "application/json",
+                Connection: "close",
+            });
+            res.end(JSON.stringify({
+                error: by === "sidecar" ? detail : "model proxy unreachable",
+            }));
         }
-        record();
+        req.unpipe(out);
+        req.pause();
+        out.destroy();
+        record(extra);
     };
     req.on("data", (chunk) => {
-        if (requestBytes < SCAN_LIMIT)
-            scanned.push(chunk);
+        if (recorded)
+            return;
         requestBytes += chunk.length;
+        if (!buffered.append(chunk)) {
+            fail("buffer_limit", "too many buffered request bytes", "sidecar", chunk);
+            return;
+        }
     });
     const out = send(target, {
         method,
@@ -162,13 +193,30 @@ function relay(context, req, res) {
     });
     out.on("timeout", () => {
         fail("timeout", `model proxy sent nothing for ${context.idleTimeoutMs} ms`);
-        out.destroy();
     });
     out.on("error", (cause) => fail("network_error", cause.message));
+    const timer = setTimeout(() => fail("request_timeout", "request body timed out", "sidecar"), context.requestTimeoutMs);
+    timer.unref();
+    req.once("end", () => clearTimeout(timer));
+    req.once("aborted", () => fail("request_aborted", "request body aborted", "sidecar"));
+    req.once("error", () => fail("request_aborted", "request body failed", "sidecar"));
     res.on("close", () => {
-        if (!res.writableFinished)
-            out.destroy();
+        clearTimeout(timer);
+        req.unpipe(out);
+        req.pause();
+        out.destroy();
+        if (!error && (!req.complete || !res.writableFinished)) {
+            error = {
+                by: "sidecar",
+                code: "request_aborted",
+                detail: req.complete ? "response aborted" : "request body aborted",
+                call: { url: target.href, status, durationMs: Date.now() - started },
+            };
+        }
         record();
+        if (!req.complete)
+            req.destroy();
+        lease.release();
     });
     req.pipe(out);
 }

package/dist/sidecar/sidecar.d.ts

@@ -1,5 +1,6 @@
 import { type SidecarPorts } from "./config.js";
 import { type Ips } from "./dns.js";
+import { type RequestLimits } from "./http.js";
 import { SidecarState } from "./state.js";
 export type SidecarOptions = {
     controlSecret: string;
@@ -11,6 +12,8 @@ export type SidecarOptions = {
     fetch?: typeof fetch;
     checkTimeoutMs?: number;
     upstreamIdleTimeoutMs?: number;
+    requestLimits?: Partial<RequestLimits>;
+    requestTimeoutMs?: number;
     lease?: {
         ms: number;
         onExpired: () => void;

package/dist/sidecar/sidecar.js

@@ -5,6 +5,7 @@ import {} from "./config.js";
 import { createControl } from "./control.js";
 import { createDns } from "./dns.js";
 import { createFakeHttp, createFakeHttps, generateTls } from "./fake-web.js";
+import { REQUEST_TIMEOUT_MS } from "./http.js";
 import { createRelay, UPSTREAM_IDLE_TIMEOUT_MS } from "./relay.js";
 import { SidecarState } from "./state.js";
 import { CHECK_TIMEOUT_MS, createToolHook } from "./tool-hook.js";
@@ -34,7 +35,7 @@ const close = (server) => new Promise((resolve) => {
     server.closeAllConnections();
 });
 export async function startSidecar(options) {
-    const state = new SidecarState();
+    const state = new SidecarState(options.requestLimits);
     const host = options.host ?? "0.0.0.0";
     const trustTestSourceHeader = options.trustTestSourceHeader ?? false;
     const { domain } = options;
@@ -49,7 +50,8 @@ export async function startSidecar(options) {
         }, Math.max(1, Math.floor(lease.ms / 4)))
         : undefined;
     leaseTimer?.unref();
-    const web = { state, domain, trustTestSourceHeader };
+    const requestTimeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_MS;
+    const web = { state, domain, trustTestSourceHeader, requestTimeoutMs };
     const servers = {
         http: createFakeHttp(web),
         https: createFakeHttps(web, tls),
@@ -57,6 +59,7 @@ export async function startSidecar(options) {
             state,
             trustTestSourceHeader,
             idleTimeoutMs: options.upstreamIdleTimeoutMs ?? UPSTREAM_IDLE_TIMEOUT_MS,
+            requestTimeoutMs,
         }),
         hook: createToolHook({
             state,
@@ -64,6 +67,7 @@ export async function startSidecar(options) {
             signal: controller.signal,
             trustTestSourceHeader,
             timeoutMs: options.checkTimeoutMs ?? CHECK_TIMEOUT_MS,
+            requestTimeoutMs,
         }),
         control: createControl({
             state,

package/dist/sidecar/state.d.ts

@@ -1,4 +1,5 @@
 import { type IncomingMessage } from "node:http";
+import { RequestBudget, type RequestLimits } from "./http.js";
 import { type Hit, type ModelCall, type ModelProxy, type SecretMatch, type Secrets, type ToolCall, type ToolCallCheck } from "../types.js";
 export declare const MAX_ENTRIES = 10000;
 export declare const MAX_LOG_BYTES: number;
@@ -37,6 +38,8 @@ export declare function drain<T>(log: Log<T>): Log<T>;
 export declare class SidecarState {
     readonly runs: Map<string, RunState>;
     readonly unattributed: Log<Hit>;
+    readonly requests: RequestBudget;
+    constructor(limits?: Partial<RequestLimits>);
     register(id: string, run: Pick<RunState, "ip" | "pages" | "secrets" | "modelProxy" | "toolCallCheck">): "ok" | "run" | "ip";
     runEntryForIp(ip: string): [string, RunState] | undefined;
     runForIp(ip: string): RunState | undefined;

package/dist/sidecar/state.js

@@ -1,5 +1,6 @@
 import {} from "node:http";
 import { findSecrets } from "../secrets.js";
+import { RequestBudget } from "./http.js";
 import {} from "../types.js";
 export const MAX_ENTRIES = 10_000;
 export const MAX_LOG_BYTES = 4 * 1024 * 1024;
@@ -61,6 +62,10 @@ export function drain(log) {
 export class SidecarState {
     runs = new Map();
     unattributed = emptyLog();
+    requests;
+    constructor(limits) {
+        this.requests = new RequestBudget(limits);
+    }
     register(id, run) {
         if (this.runs.has(id))
             return "run";

package/dist/sidecar/tool-hook.d.ts

@@ -9,5 +9,6 @@ export type ToolHookContext = {
     signal: AbortSignal;
     trustTestSourceHeader: boolean;
     timeoutMs: number;
+    requestTimeoutMs: number;
 };
 export declare function createToolHook(context: ToolHookContext): Server;

package/dist/sidecar/tool-hook.js

@@ -2,7 +2,7 @@ import {} from "node:http";
 import { z } from "zod";
 import { findSecrets } from "../secrets.js";
 import { DETAIL_LIMIT, isRecord, messageOf, } from "../types.js";
-import { BodyTooLargeError, jsonServer, readBody, readJson, sendJson, } from "./http.js";
+import { BodyTooLargeError, BODY_LIMIT_BYTES, jsonServer, readBody, readJson, sendJson, } from "./http.js";
 import { push, sourceIp } from "./state.js";
 export const CHECK_TIMEOUT_MS = 25_000;
 const hookSchema = z.object({
@@ -111,37 +111,86 @@ export function createToolHook(context) {
                 at: new Date(started).toISOString(),
                 error: { by: "sidecar", code, detail: detail.slice(0, DETAIL_LIMIT) },
             });
-            sendJson(res, status, {});
+            if (!res.destroyed)
+                sendJson(res, status, {}, { Connection: "close" });
         };
-        let payload;
+        const lease = context.state.requests.take(source);
+        if (!lease)
+            return refuse(429, "request_limit", "too many in-flight requests");
+        const controller = new AbortController();
+        let handlerDone = false;
+        let responseDone = false;
+        let timedOut = false;
+        const release = () => {
+            if (handlerDone && responseDone)
+                lease.release();
+        };
+        res.once("finish", () => {
+            responseDone = true;
+            release();
+        });
+        res.once("close", () => {
+            responseDone = true;
+            controller.abort();
+            release();
+        });
+        req.once("aborted", () => {
+            controller.abort();
+        });
+        req.once("error", () => {
+            controller.abort();
+        });
+        const timer = setTimeout(() => {
+            timedOut = true;
+            req.destroy(new Error("request body timed out"));
+        }, context.requestTimeoutMs);
+        timer.unref();
+        req.once("end", () => clearTimeout(timer));
         try {
-            payload = await readJson(req);
-        }
-        catch (error) {
-            return error instanceof BodyTooLargeError
-                ? refuse(413, "payload_too_large", messageOf(error))
-                : refuse(400, "invalid_payload", messageOf(error));
+            if (!lease.reserve(2 * BODY_LIMIT_BYTES)) {
+                return refuse(429, "buffer_limit", "too many buffered request bytes");
+            }
+            let payload;
+            try {
+                payload = await readJson(req);
+            }
+            catch (error) {
+                if (timedOut)
+                    return refuse(408, "request_timeout", "request body timed out");
+                return error instanceof BodyTooLargeError
+                    ? refuse(413, "payload_too_large", messageOf(error))
+                    : refuse(400, "invalid_payload", messageOf(error));
+            }
+            const hook = hookSchema.safeParse(payload);
+            if (!hook.success) {
+                return refuse(400, "invalid_payload", hook.error.message, payload);
+            }
+            const check = owned.toolCallCheck;
+            const checked = check
+                ? await askCheck({
+                    ...context,
+                    signal: AbortSignal.any([context.signal, controller.signal]),
+                }, check, runId, payload)
+                : { response: {} };
+            push(owned.toolCalls, {
+                toolUseId: hook.data.tool_use_id,
+                tool: hook.data.tool_name,
+                input: hook.data.tool_input,
+                hookPayload: payload,
+                response: checked.response,
+                action: actionOf(checked.response),
+                checked: check !== undefined,
+                latencyMs: Date.now() - started,
+                at: new Date(started).toISOString(),
+                ...(checked.error ? { error: checked.error } : {}),
+            }, findSecrets(String(JSON.stringify(hook.data.tool_input)), owned.secrets));
+            if (!res.destroyed)
+                sendJson(res, 200, checked.response);
         }
-        const hook = hookSchema.safeParse(payload);
-        if (!hook.success) {
-            return refuse(400, "invalid_payload", hook.error.message, payload);
+        finally {
+            clearTimeout(timer);
+            handlerDone = true;
+            release();
         }
-        const check = owned.toolCallCheck;
-        const checked = check
-            ? await askCheck(context, check, runId, payload)
-            : { response: {} };
-        push(owned.toolCalls, {
-            toolUseId: hook.data.tool_use_id,
-            tool: hook.data.tool_name,
-            input: hook.data.tool_input,
-            hookPayload: payload,
-            response: checked.response,
-            action: actionOf(checked.response),
-            checked: check !== undefined,
-            latencyMs: Date.now() - started,
-            at: new Date(started).toISOString(),
-            ...(checked.error ? { error: checked.error } : {}),
-        }, findSecrets(String(JSON.stringify(hook.data.tool_input)), owned.secrets));
-        sendJson(res, 200, checked.response);
     });
 }