mslxdff 0.1.28 → 0.1.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.28",
+  "version": "0.1.30",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/routes.js

@@ -84,19 +84,36 @@ async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_
     res.setHeader("Cache-Control", "no-cache");
     res.setHeader("Connection", "keep-alive");
     let ttf = null;
+    let interrupted = false;
     if (upRes.body) {
       let first = true;
       let wroteAny = false;
       let timedOut = false;
-      const timer = setTimeout(() => {
+      let stalled = false;
+      let tooLong = false;
+      let stallTimer = null;
+      const armStall = () => {
+        if (stallTimer) clearTimeout(stallTimer);
+        stallTimer = STALL_TIMEOUT_MS
+          ? setTimeout(() => {
+              stalled = true;
+              if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
+            }, STALL_TIMEOUT_MS)
+          : null;
+      };
+      const firstTimer = setTimeout(() => {
         timedOut = true;
-        // nothing written yet — cancel the upstream body so the loop can exit
-        // and we can fail over to the next model cleanly.
         if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
       }, streamTimeoutMs);
+      const maxTimer = MAX_STREAM_MS
+        ? setTimeout(() => {
+            tooLong = true;
+            if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
+          }, MAX_STREAM_MS)
+        : null;
       try {
         for await (const chunk of upRes.body) {
-          if (timedOut) break;
+          if (timedOut || stalled || tooLong) break;
           if (first) {
             first = false;
             ttf = Math.round(performance.now() - t0);
@@ -104,22 +121,29 @@ async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_
           }
           wroteAny = true;
           res.write(chunk);
+          armStall(); // any fresh chunk resets the stall clock
         }
       } catch (err) {
-        timedOut = true;
+        // body threw — if nothing was written, treat it as a first-block timeout
+        if (!wroteAny) timedOut = true;
+        else stalled = true;
       } finally {
-        clearTimeout(timer);
+        if (firstTimer) clearTimeout(firstTimer);
+        if (maxTimer) clearTimeout(maxTimer);
+        if (stallTimer) clearTimeout(stallTimer);
       }
       if (timedOut && !wroteAny) {
         // nothing written to res yet — safe to drop this model and let the
         // caller fail over to the next one. Do NOT write/end res here.
         return { status: STREAM_TIMEOUT_MS, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true };
       }
-      if (timedOut && wroteAny) {
-        // we'd already started streaming when it died — can't fail over, just
-        // end the response so the client sees a clean EOF.
+      if ((stalled || tooLong) && wroteAny) {
+        // a response that was flowing either went silent for the stall window
+        // or blew the total ceiling — we can't cleanly fail over a half-written
+        // body, so end it and let the caller remember this model as slow.
+        interrupted = true;
         try { res.end(); } catch { /* ignore */ }
-        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false };
+        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted };
       }
     }
     const totalMs = Math.round(performance.now() - t0);
@@ -255,6 +279,23 @@ export const STREAM_TIMEOUT_MS = (() => {
   return Number.isInteger(n) && n > 0 ? n : 25_000;
 })();
 
+// Ceiling on how long a streamed response may produce no new chunk before we
+// treat the model as stalled and proactively end the stream (clean EOF instead
+// of hanging on a dead upstream). A model that keeps emitting chunks is never
+// cut off — only silence triggers it. Set to 0 to disable.
+export const STALL_TIMEOUT_MS = (() => {
+  const n = Number(process.env.MSLXDFF_STALL_TIMEOUT_MS);
+  return Number.isInteger(n) && n > 0 ? n : 15_000;
+})();
+
+// Loose total ceiling (TTFB + generation) so an unbounded stream can never
+// run forever. Kept much larger than the stall timeout so normally-flowing
+// responses are not truncated. Set to 0 to disable.
+export const MAX_STREAM_MS = (() => {
+  const n = Number(process.env.MSLXDFF_MAX_STREAM_MS);
+  return Number.isInteger(n) && n > 0 ? n : 120_000;
+})();
+
 async function racePeerCandidates(candidates, ctx) {
   for (let i = 0; i < candidates.length; i += PEER_RACE_LIMIT) {
     const batch = candidates.slice(i, i + PEER_RACE_LIMIT);
@@ -414,6 +455,17 @@ const ROUTES = [
             upRes = null;
             continue;
           }
+          if (out.interrupted) {
+            // the model went silent past STALL_TIMEOUT_MS (or blew the total
+            // ceiling) — we ended it so the client got a clean EOF instead of
+            // hanging. Remember this model as slow so the next request prefers
+            // a faster one.
+            if (auto) await auto.recordError(model, { status: 200, slow: true, note: `stall ${STALL_TIMEOUT_MS}ms` });
+            evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: STALL_TIMEOUT_MS, interrupted: true });
+            logCall(model, 200);
+            evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true });
+            return;
+          }
           // A model that took a long wall-clock time (TTFB + generation + relay)
           // gets remembered as slow so the next request prefers a faster one.
           const elapsed = Date.now() - startedAt;