mslxdff 0.1.27 → 0.1.29

package/bin/mslxdff.js

@@ -587,6 +587,7 @@ const models = createModelsService({
 });
 const auto = createAutoSelector({
   cooldownMs: modelCooldownMs(),
+  slowCooldownMs: slowCooldownMs(),
   loadCandidates: async () => {
     try {
       return (await models.get()).data.map((m) => m.id);
@@ -686,6 +687,11 @@ function modelCooldownMs() {
   return Number.isInteger(n) && n > 0 ? n : 60_000;
 }
 
+function slowCooldownMs() {
+  const n = Number(process.env.MSLXDFF_SLOW_COOLDOWN_MS);
+  return Number.isInteger(n) && n > 0 ? n : 5 * 60_000;
+}
+
 function peerCooldownMs() {
   const n = Number(process.env.MSLXDFF_PEER_COOLDOWN_MS);
   return Number.isInteger(n) && n > 0 ? n : 30_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.27",
+  "version": "0.1.29",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/auto.js

@@ -22,19 +22,23 @@ export const MODEL_STATUS = Object.freeze({
 
 // Legacy modelErrors entries are bare timestamps ({id: ts}); newer ones are
 // objects ({id: {status, at, code}}). Normalize both to an entry object.
+// `slow` flags a model whose last request was slow (over the wall-clock
+// threshold) — those get a longer cooldown so they lie low until they recover.
 function normEntry(e) {
-  if (typeof e === "number") return { status: MODEL_STATUS.ERROR, at: e, code: null };
+  if (typeof e === "number") return { status: MODEL_STATUS.ERROR, at: e, code: null, slow: false };
   if (e && typeof e === "object") {
     return {
       status: e.status || MODEL_STATUS.ERROR,
       at: typeof e.at === "number" ? e.at : 0,
       code: e.code ?? null,
+      slow: Boolean(e.slow),
     };
   }
   return null;
 }
 
 export function classifyErrorEvent(evt = {}) {
+  if (evt.slow) return MODEL_STATUS.ERROR;
   const code = Number(evt.status);
   if (code === 429) return MODEL_STATUS.LIMIT;
   const msg = String(evt.message || evt.note || "").toLowerCase();
@@ -45,21 +49,29 @@ export function classifyErrorEvent(evt = {}) {
 }
 
 export const DEFAULT_COOLDOWN_MS = 60_000;
+export const DEFAULT_SLOW_COOLDOWN_MS = 5 * 60_000;
 
-function inCooldown(id, errors, now, cooldownMs) {
-  if (!cooldownMs) return false;
-  const at = normEntry(errors[id])?.at ?? 0;
-  return at > 0 && now - at < cooldownMs;
+function effectiveCooldown(entry, slowCooldownMs, cooldownMs) {
+  if (entry && entry.slow) return slowCooldownMs || 0;
+  return cooldownMs || 0;
 }
 
-export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0 } = {}) {
+function inCooldown(id, errors, now, cooldownMs, slowCooldownMs) {
+  const e = normEntry(errors[id]);
+  if (!e || !(e.at > 0)) return false;
+  const cd = effectiveCooldown(e, slowCooldownMs, cooldownMs);
+  return cd > 0 && now - e.at < cd;
+}
+
+export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0, slowCooldownMs = 0 } = {}) {
   return [...new Set(ids)]
     .filter(Boolean)
     .map((id) => ({
       id,
+      e: normEntry(errors[id]),
       err: normEntry(errors[id])?.at ?? 0,
       isDeepseek: /deepseek/i.test(id),
-      cooling: inCooldown(id, errors, now, cooldownMs),
+      cooling: inCooldown(id, errors, now, cooldownMs, slowCooldownMs),
     }))
     .sort(
       (a, b) =>
@@ -75,6 +87,7 @@ export function createAutoSelector({
   file,
   now = () => Date.now(),
   cooldownMs = DEFAULT_COOLDOWN_MS,
+  slowCooldownMs = DEFAULT_SLOW_COOLDOWN_MS,
   errors: seedErrors,
   persist = (errors, f = file) => saveModelErrors(errors, f ? { file: f } : {}),
 } = {}) {
@@ -92,7 +105,7 @@ export function createAutoSelector({
   }
 
   async function candidates() {
-    return rankModels(await loadList(), lastErrorAt, { now: now(), cooldownMs });
+    return rankModels(await loadList(), lastErrorAt, { now: now(), cooldownMs, slowCooldownMs });
   }
 
   async function candidatesFor(requested) {
@@ -102,15 +115,16 @@ export function createAutoSelector({
     const others = rankModels(all.filter((id) => id !== requested), lastErrorAt, {
       now: now(),
       cooldownMs,
+      slowCooldownMs,
     });
-    if (inCooldown(requested, lastErrorAt, now(), cooldownMs)) {
+    if (inCooldown(requested, lastErrorAt, now(), cooldownMs, slowCooldownMs)) {
       return [...others, requested];
     }
     return [requested, ...others];
   }
 
   function isCooling(id) {
-    return inCooldown(id, lastErrorAt, now(), cooldownMs);
+    return inCooldown(id, lastErrorAt, now(), cooldownMs, slowCooldownMs);
   }
 
   async function recordError(id, evt = {}) {
@@ -119,13 +133,14 @@ export function createAutoSelector({
       status: classifyErrorEvent(evt),
       at: now(),
       code: Number.isInteger(Number(evt.status)) ? Number(evt.status) : null,
+      slow: Boolean(evt.slow),
     };
     await persist({ ...lastErrorAt });
   }
 
   async function recordOk(id) {
     if (!id) return;
-    lastErrorAt[id] = { status: MODEL_STATUS.NORMAL, at: now(), code: 200 };
+    lastErrorAt[id] = { status: MODEL_STATUS.NORMAL, at: now(), code: 200, slow: false };
     await persist({ ...lastErrorAt });
   }
 

package/src/routes.js

@@ -84,19 +84,27 @@ async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_
     res.setHeader("Cache-Control", "no-cache");
     res.setHeader("Connection", "keep-alive");
     let ttf = null;
+    let interrupted = false;
     if (upRes.body) {
       let first = true;
       let wroteAny = false;
       let timedOut = false;
-      const timer = setTimeout(() => {
+      // whole-stream ceiling (TTFB + generation): cancels the upstream body so
+      // the loop exits and we proactively end the stream instead of hanging.
+      let genTooLong = false;
+      const genTimer = GEN_TIMEOUT_MS
+        ? setTimeout(() => {
+            genTooLong = true;
+            if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
+          }, GEN_TIMEOUT_MS)
+        : null;
+      const firstTimer = setTimeout(() => {
         timedOut = true;
-        // nothing written yet — cancel the upstream body so the loop can exit
-        // and we can fail over to the next model cleanly.
         if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
       }, streamTimeoutMs);
       try {
         for await (const chunk of upRes.body) {
-          if (timedOut) break;
+          if (timedOut || genTooLong) break;
           if (first) {
             first = false;
             ttf = Math.round(performance.now() - t0);
@@ -106,20 +114,25 @@ async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_
           res.write(chunk);
         }
       } catch (err) {
-        timedOut = true;
+        // body threw — if we already started, treat as a mid-stream interrupt
+        if (!wroteAny) { timedOut = true; genTooLong = false; }
       } finally {
-        clearTimeout(timer);
+        if (firstTimer) clearTimeout(firstTimer);
+        if (genTimer) clearTimeout(genTimer);
       }
       if (timedOut && !wroteAny) {
         // nothing written to res yet — safe to drop this model and let the
         // caller fail over to the next one. Do NOT write/end res here.
         return { status: STREAM_TIMEOUT_MS, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true };
       }
-      if (timedOut && wroteAny) {
-        // we'd already started streaming when it died — can't fail over, just
-        // end the response so the client sees a clean EOF.
+      if (genTooLong || (timedOut && wroteAny)) {
+        // we'd already started streaming when it exceeded the ceiling — can't
+        // cleanly fail over a half-written response, just end it so the client
+        // sees a clean EOF rather than hanging. interrupted signals the caller
+        // to remember this model as slow.
+        interrupted = true;
         try { res.end(); } catch { /* ignore */ }
-        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false };
+        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted };
       }
     }
     const totalMs = Math.round(performance.now() - t0);
@@ -244,7 +257,7 @@ export const PEER_RACE_LIMIT = Number(process.env.MSLXDFF_PEER_RACE_LIMIT) > 0
 // Set MSLXDFF_SLOW_TOTAL_MS=0 to disable.
 export const SLOW_TOTAL_MS = (() => {
   const n = Number(process.env.MSLXDFF_SLOW_TOTAL_MS);
-  return Number.isInteger(n) && n > 0 ? n : 15_000;
+  return Number.isInteger(n) && n > 0 ? n : 20_000;
 })();
 
 // How long to wait for the first chunk of a streamed response before giving up
@@ -255,6 +268,15 @@ export const STREAM_TIMEOUT_MS = (() => {
   return Number.isInteger(n) && n > 0 ? n : 25_000;
 })();
 
+// Ceiling on the total wall-clock a single streamed response may run (TTFB +
+// generation + relay). Once exceeded we proactively end the stream so the
+// client gets a clean EOF instead of hanging on a very slow model; the model
+// is then flagged slow and demoted for the next request. Set to 0 to disable.
+export const GEN_TIMEOUT_MS = (() => {
+  const n = Number(process.env.MSLXDFF_GEN_TIMEOUT_MS);
+  return Number.isInteger(n) && n > 0 ? n : 20_000;
+})();
+
 async function racePeerCandidates(candidates, ctx) {
   for (let i = 0; i < candidates.length; i += PEER_RACE_LIMIT) {
     const batch = candidates.slice(i, i + PEER_RACE_LIMIT);
@@ -414,6 +436,16 @@ const ROUTES = [
             upRes = null;
             continue;
           }
+          if (out.interrupted) {
+            // the whole stream ran past GEN_TIMEOUT_MS — we ended it proactively
+            // so the client got a clean EOF instead of hanging. Remember this
+            // model as slow so the next request prefers a faster one.
+            if (auto) await auto.recordError(model, { status: 200, slow: true, note: `gen timeout ${GEN_TIMEOUT_MS}ms` });
+            evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: GEN_TIMEOUT_MS, interrupted: true });
+            logCall(model, 200);
+            evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true });
+            return;
+          }
           // A model that took a long wall-clock time (TTFB + generation + relay)
           // gets remembered as slow so the next request prefers a faster one.
           const elapsed = Date.now() - startedAt;