@yzj01/llm-router 1.0.0 → 1.0.1

package/README.md

@@ -130,6 +130,20 @@ Unsupported model IDs return HTTP 400. Supported request IDs are currently only
 `auto`. Configured aliases remain internal routing outputs and are surfaced via
 response headers, not accepted in request bodies.
 
+## Runtime Protections
+
+The local proxy applies conservative runtime limits by default. These limits are
+internal defaults rather than public config fields:
+
+| Limit | Default | Behavior |
+| --- | ---: | --- |
+| Request body size | 10 MB | Returns `413 Payload Too Large` before forwarding upstream. |
+| Request body read time | 30 s | Returns `408 Request Timeout` if the client does not finish sending the body. |
+| Upstream request time | 300 s | Aborts the upstream request and returns `504 Gateway Timeout`. |
+
+If the client disconnects before the upstream call completes, the proxy aborts
+the upstream request to avoid keeping stale work running.
+
 ## Response Headers
 
 The proxy adds routing headers:

package/dist/cli.d.ts

@@ -1,4 +1,4 @@
-import { P as ProxyOptions, a as ProxyHandle } from './proxy-CrRX9deF.js';
+import { P as ProxyOptions, a as ProxyHandle } from './proxy-Co87mKak.js';
 
 type CliRuntime = {
     log: (msg: string) => void;

package/dist/cli.js

@@ -1792,7 +1792,7 @@ function hashHex(value, length) {
 }
 
 // src/proxy.ts
-var VERSION = "1.0.0";
+var VERSION = "1.0.1";
 var HOP_BY_HOP = /* @__PURE__ */ new Set([
   "connection",
   "keep-alive",
@@ -1806,17 +1806,85 @@ var HOP_BY_HOP = /* @__PURE__ */ new Set([
   "content-length"
 ]);
 var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
+var DEFAULT_MAX_BODY_BYTES = 10 * 1024 * 1024;
+var DEFAULT_BODY_READ_TIMEOUT_MS = 3e4;
+var DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS = 3e5;
+var DEFAULT_RUNTIME_LIMITS = {
+  maxBodyBytes: DEFAULT_MAX_BODY_BYTES,
+  bodyReadTimeoutMs: DEFAULT_BODY_READ_TIMEOUT_MS,
+  upstreamRequestTimeoutMs: DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS
+};
+function resolveRuntimeLimits(input) {
+  return {
+    maxBodyBytes: input?.maxBodyBytes ?? DEFAULT_RUNTIME_LIMITS.maxBodyBytes,
+    bodyReadTimeoutMs: input?.bodyReadTimeoutMs ?? DEFAULT_RUNTIME_LIMITS.bodyReadTimeoutMs,
+    upstreamRequestTimeoutMs: input?.upstreamRequestTimeoutMs ?? DEFAULT_RUNTIME_LIMITS.upstreamRequestTimeoutMs
+  };
+}
 var REQUESTABLE_PUBLIC_MODELS = /* @__PURE__ */ new Set(["auto"]);
 var PUBLIC_HEADER_PREFIXES = ["x-xy-router-"];
-function readBody(req) {
+var BodyReadError = class extends Error {
+  statusCode;
+  constructor(statusCode, message) {
+    super(message);
+    this.name = "BodyReadError";
+    this.statusCode = statusCode;
+  }
+};
+function isBodyReadError(error) {
+  return error instanceof BodyReadError;
+}
+function writeBodyReadErrorAndCloseRequest(req, res, error) {
+  res.shouldKeepAlive = false;
+  res.setHeader("connection", "close");
+  res.once("finish", () => {
+    if (!req.destroyed) {
+      req.destroy();
+    }
+  });
+  writeOpenAiError(res, error.statusCode, error.message);
+}
+function readBody(req, limits) {
   return new Promise((resolve, reject) => {
     let body = "";
+    let bodyBytes = 0;
+    let settled = false;
     req.setEncoding("utf8");
-    req.on("data", (chunk) => {
+    const timeout = setTimeout(() => {
+      rejectOnce(new BodyReadError(408, "Request body read timeout"));
+    }, limits.bodyReadTimeoutMs);
+    const cleanup = () => {
+      clearTimeout(timeout);
+      req.off("data", onData);
+      req.off("end", onEnd);
+      req.off("error", onError);
+    };
+    const rejectOnce = (error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(error);
+    };
+    const resolveOnce = (value) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve(value);
+    };
+    const onData = (chunk) => {
+      if (settled) return;
+      bodyBytes += Buffer.byteLength(chunk, "utf8");
+      if (bodyBytes > limits.maxBodyBytes) {
+        rejectOnce(new BodyReadError(413, "Payload Too Large"));
+        return;
+      }
       body += chunk;
-    });
-    req.on("end", () => resolve(body));
-    req.on("error", reject);
+    };
+    const onEnd = () => resolveOnce(body);
+    const onError = (error) => rejectOnce(error);
+    req.on("data", onData);
+    req.on("end", onEnd);
+    req.on("error", onError);
   });
 }
 var OPENCLAW_CLI_TURN_PATTERN = /(?:^|\n)\[[^\]\n]+?\]\s+([\s\S]*?)(?=(?:\n\[[^\]\n]+?\]\s+)|$)/g;
@@ -1931,19 +1999,68 @@ function writeOpenAiError(res, status, message, type = "invalid_request_error",
     }
   });
 }
-async function fetchUpstream(cfg, req, body, actualModel) {
+function anySignal(signals) {
+  const controller = new AbortController();
+  const listeners = [];
+  let cleaned = false;
+  const cleanup = () => {
+    if (cleaned) return;
+    cleaned = true;
+    for (const { signal, listener } of listeners) {
+      signal.removeEventListener("abort", listener);
+    }
+  };
+  const abortFrom = (signal) => {
+    if (!controller.signal.aborted) {
+      controller.abort(signal.reason);
+    }
+    cleanup();
+  };
+  for (const signal of signals) {
+    if (signal.aborted) {
+      abortFrom(signal);
+      return { signal: controller.signal, cleanup };
+    }
+  }
+  for (const signal of signals) {
+    const listener = () => {
+      abortFrom(signal);
+    };
+    listeners.push({ signal, listener });
+    signal.addEventListener("abort", listener, { once: true });
+  }
+  return { signal: controller.signal, cleanup };
+}
+async function fetchUpstream(cfg, req, body, actualModel, runtimeLimits, requestSignal) {
+  const timeoutController = new AbortController();
+  const timeout = setTimeout(() => {
+    timeoutController.abort();
+  }, runtimeLimits.upstreamRequestTimeoutMs);
+  const linkedSignal = anySignal([requestSignal, timeoutController.signal]);
+  const cleanup = () => {
+    clearTimeout(timeout);
+    linkedSignal.cleanup();
+  };
   try {
     const response = await fetch(`${cfg.baseUrl}/chat/completions`, {
       method: "POST",
       headers: buildUpstreamHeaders(req, cfg),
-      body: JSON.stringify({ ...body, model: actualModel })
+      body: JSON.stringify({ ...body, model: actualModel }),
+      signal: linkedSignal.signal
     });
+    clearTimeout(timeout);
     if (RETRYABLE_STATUS.has(response.status)) {
-      return { ok: false, reason: "retryable", response };
+      return { ok: false, reason: "retryable", response, cleanup };
     }
-    return { ok: true, response };
+    return { ok: true, response, cleanup };
   } catch (error) {
-    return { ok: false, reason: "network_error", error };
+    if (timeoutController.signal.aborted) {
+      return { ok: false, reason: "timeout", error, cleanup };
+    }
+    if (requestSignal.aborted) {
+      return { ok: false, reason: "aborted", error, cleanup };
+    }
+    return { ok: false, reason: "network_error", error, cleanup };
   }
 }
 function getMaxOutputTokens(body) {
@@ -2159,8 +2276,17 @@ function chooseModel(requestedModel, body, headers, sessionStore, cfg, tierEntri
     sessionAction: "none"
   };
 }
-async function proxyChat(req, res, cfg, sessionStore, tierEntries, publicModels, registry2, routerOptions) {
-  const rawBody = await readBody(req);
+async function proxyChat(req, res, cfg, runtimeLimits, sessionStore, tierEntries, publicModels, registry2, routerOptions) {
+  let rawBody;
+  try {
+    rawBody = await readBody(req, runtimeLimits);
+  } catch (error) {
+    if (isBodyReadError(error)) {
+      writeBodyReadErrorAndCloseRequest(req, res, error);
+      return;
+    }
+    throw error;
+  }
   let body;
   try {
     body = JSON.parse(rawBody);
@@ -2205,61 +2331,129 @@ async function proxyChat(req, res, cfg, sessionStore, tierEntries, publicModels,
     );
     return;
   }
-  const attempt = await fetchUpstream(cfg, req, bodyObj, physicalModel.id);
-  const attempts = [
-    attempt.ok ? { model: selected.actualModel, status: "success" } : {
-      model: selected.actualModel,
-      status: "error",
-      error: attempt.reason === "network_error" ? "network_error" : `upstream_http_${attempt.response.status}`
+  const requestController = new AbortController();
+  let responseFinished = false;
+  const abortUpstreamRequest = () => {
+    if (responseFinished || requestController.signal.aborted) return;
+    requestController.abort();
+  };
+  const onRequestAborted = () => {
+    abortUpstreamRequest();
+  };
+  const onResponseClose = () => {
+    abortUpstreamRequest();
+  };
+  const onResponseFinish = () => {
+    responseFinished = true;
+  };
+  const cleanupRequestAbortListeners = () => {
+    req.off("aborted", onRequestAborted);
+    res.off("close", onResponseClose);
+    res.off("finish", onResponseFinish);
+  };
+  req.on("aborted", onRequestAborted);
+  res.on("close", onResponseClose);
+  res.on("finish", onResponseFinish);
+  let attempt;
+  try {
+    attempt = await fetchUpstream(
+      cfg,
+      req,
+      bodyObj,
+      physicalModel.id,
+      runtimeLimits,
+      requestController.signal
+    );
+    const attempts = [
+      attempt.ok ? { model: selected.actualModel, status: "success" } : {
+        model: selected.actualModel,
+        status: "error",
+        error: attempt.reason === "timeout" ? "upstream_timeout" : attempt.reason === "network_error" ? "network_error" : attempt.reason === "aborted" ? "client_aborted" : `upstream_http_${attempt.response.status}`
+      }
+    ];
+    const finalTier = selected.tier;
+    let sessionAction = selected.sessionAction;
+    if (!attempt.ok && attempt.reason === "aborted") {
+      emitProxyTrace(
+        cfg,
+        selected,
+        finalTier,
+        attempts,
+        sessionAction,
+        true
+      );
+      return;
     }
-  ];
-  const finalTier = selected.tier;
-  let sessionAction = selected.sessionAction;
-  if (!attempt.ok && attempt.reason === "network_error") {
-    const trace2 = emitProxyTrace(
+    if (!attempt.ok && attempt.reason === "timeout") {
+      const trace2 = emitProxyTrace(
+        cfg,
+        selected,
+        finalTier,
+        attempts,
+        sessionAction,
+        true
+      );
+      const headers2 = buildPublicHeaders(cfg, selected, finalTier, trace2);
+      writeOpenAiError(
+        res,
+        504,
+        "Upstream request timed out",
+        "invalid_request_error",
+        null,
+        headers2
+      );
+      return;
+    }
+    if (!attempt.ok && attempt.reason === "network_error") {
+      const trace2 = emitProxyTrace(
+        cfg,
+        selected,
+        finalTier,
+        attempts,
+        sessionAction,
+        true
+      );
+      const headers2 = buildPublicHeaders(cfg, selected, finalTier, trace2);
+      writeOpenAiError(
+        res,
+        502,
+        attempt.error instanceof Error ? attempt.error.message : "Upstream request failed",
+        "invalid_request_error",
+        null,
+        headers2
+      );
+      return;
+    }
+    if (attempt.ok && selected.sessionId && !selected.explicit) {
+      sessionStore.setSession(selected.sessionId, {
+        physicalModelId: selected.actualModel,
+        routedPublicModel: selected.routedModel,
+        pinnedTier: finalTier
+      });
+      if (sessionAction === "none") {
+        sessionAction = "set";
+      }
+    }
+    const trace = emitProxyTrace(
       cfg,
       selected,
       finalTier,
       attempts,
       sessionAction,
-      true
-    );
-    const headers2 = buildPublicHeaders(cfg, selected, finalTier, trace2);
-    writeOpenAiError(
-      res,
-      502,
-      attempt.error instanceof Error ? attempt.error.message : "Upstream request failed",
-      "invalid_request_error",
-      null,
-      headers2
+      !attempt.ok
     );
-    return;
-  }
-  if (attempt.ok && selected.sessionId && !selected.explicit) {
-    sessionStore.setSession(selected.sessionId, {
-      physicalModelId: selected.actualModel,
-      routedPublicModel: selected.routedModel,
-      pinnedTier: finalTier
-    });
-    if (sessionAction === "none") {
-      sessionAction = "set";
+    const headers = buildPublicHeaders(cfg, selected, finalTier, trace);
+    const responseHeaders = copyResponseHeaders(attempt.response, headers);
+    res.statusCode = attempt.response.status;
+    for (const [k, v] of Object.entries(responseHeaders)) {
+      res.setHeader(k, v);
     }
+    await streamResponse(attempt.response, res);
+    responseFinished = true;
+  } finally {
+    cleanupRequestAbortListeners();
+    attempt?.cleanup();
   }
-  const trace = emitProxyTrace(
-    cfg,
-    selected,
-    finalTier,
-    attempts,
-    sessionAction,
-    !attempt.ok
-  );
-  const headers = buildPublicHeaders(cfg, selected, finalTier, trace);
-  const responseHeaders = copyResponseHeaders(attempt.response, headers);
-  res.statusCode = attempt.response.status;
-  for (const [k, v] of Object.entries(responseHeaders)) {
-    res.setHeader(k, v);
-  }
-  await streamResponse(attempt.response, res);
 }
 async function startProxy(options) {
   const cfg = resolveConfig({
@@ -2271,6 +2465,7 @@ async function startProxy(options) {
     traceLogger: options.traceLogger,
     sessionPinning: options.session?.enabled
   });
+  const runtimeLimits = resolveRuntimeLimits(options.runtimeLimits);
   const sessionStore = new SessionStore(options.session);
   const publicModels = options.config.publicModels;
   const tierEntries = options.config.routing.tiers;
@@ -2299,6 +2494,7 @@ async function startProxy(options) {
             req,
             res,
             cfg,
+            runtimeLimits,
             sessionStore,
             tierEntries,
             publicModels,