mslxdff 0.1.151 → 0.1.153

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.151",
+  "version": "0.1.153",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/chat-pipeline/serial-trial.js

@@ -10,6 +10,20 @@ import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
 import { handleViaRoute } from "../routes/chat/via-route-handler.js";
 import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
 import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
+import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// 空转重试档位（每次请求读取 env，单测可覆盖）：默认同模型最多重试 2 次、间隔 1s；
+// MSLXDFF_EMPTY_TURN_RETRIES=0 关闭（回旧行为：空转直接换候选/终结）。
+function emptyRetryCfg() {
+  const r = Number(process.env.MSLXDFF_EMPTY_TURN_RETRIES);
+  const d = Number(process.env.MSLXDFF_EMPTY_TURN_RETRY_DELAY_MS);
+  return {
+    max: Number.isInteger(r) && r >= 0 ? r : 2,
+    delayMs: Number.isFinite(d) && d >= 0 ? d : 1000,
+  };
+}
 
 function groupSkipReason(model) {
   if (isHardLocalOnly(model)) return "provider local-only（禁组员，仅本机直连）";
@@ -54,7 +68,7 @@ export async function runSerialTrial(ctx, deps = {}) {
   }
 
   let lastErr = viaRouteLastErr;
-  for (let idx = 0; idx < order.length; idx++) {
+  candidate: for (let idx = 0; idx < order.length; idx++) {
     const model = order[idx];
     handlerCtx.model = model;
     handlerCtx.orderLen = order.length;
@@ -72,82 +86,99 @@ export async function runSerialTrial(ctx, deps = {}) {
       for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
       if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") { forwarded = ur.value.payload; evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null }); }
     }
-    const tUp = performance.now();
-    evt("upstream-try", { reqId, model, attempt: idx + 1 });
-    try {
-      const chatOpts = {};
-      if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
-      if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
-      if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
-      upRes = await upstream.chat(forwarded, Object.keys(chatOpts).length ? chatOpts : undefined);
-      evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
-    } catch (err) {
-      if (auto) await auto.recordError(model, { message: errMsg(err) });
-      lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
-      logError(model, 502, errMsg(err));
-      evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
-    }
-    if (plugins?.length) {
-      runHook(plugins, "upstream:response", {
-        reqId, requested, model,
-        status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
-        ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
-        error: upRes instanceof Error ? errMsg(upRes) : null,
-        timing: upRes?._t ?? null,
-      }).catch(() => {});
-    }
-    mark(`up-${model}`);
-    if (upRes && upRes.status >= 400) {
-      const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
-      if (isAllowlistBlock) {
-        let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
-        let errBody = { error: `model not allowed for provider` };
-        try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
-        if (useAuto) {
+    const chatOpts = {};
+    if (Object.keys(shareKeys).length) chatOpts.shareKeys = shareKeys;
+    if (workbuddyUid) chatOpts.workbuddyUid = workbuddyUid;
+    if (handlerCtx?.sessionId) chatOpts.sessionId = handlerCtx.sessionId;
+    const chatOptsArg = Object.keys(chatOpts).length ? chatOpts : undefined;
+    // 空转 200（模型无输出）同模型暂停重试：默认 2 次、间隔 1s；仅 EMPTY_MODEL_RESPONSE，
+    // 429/403/500 与 fetch 异常走原有切号/failover（防烧额度）。MSLXDFF_EMPTY_TURN_RETRIES=0 关闭。
+    const emptyCfg = emptyRetryCfg();
+    let emptyRetried = 0;
+    for (;;) {
+      const tUp = performance.now();
+      evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried });
+      try {
+        upRes = await upstream.chat(forwarded, chatOptsArg);
+        evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
+      } catch (err) {
+        if (auto) await auto.recordError(model, { message: errMsg(err) });
+        lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
+        logError(model, 502, errMsg(err));
+        evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
+        upRes = null;
+      }
+      if (plugins?.length) {
+        runHook(plugins, "upstream:response", {
+          reqId, requested, model,
+          status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
+          ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
+          error: upRes instanceof Error ? errMsg(upRes) : null,
+          timing: upRes?._t ?? null,
+        }).catch(() => {});
+      }
+      mark(`up-${model}`);
+      if (upRes && upRes.status >= 400) {
+        const isAllowlistBlock = upRes.status === 403 && (upRes.headers?.get?.("x-mslxdff-allowlist") === "1");
+        if (isAllowlistBlock) {
+          let bodyText = null; try { bodyText = await upRes.clone().text(); } catch {}
+          let errBody = { error: `model not allowed for provider` };
+          try { errBody = bodyText ? JSON.parse(bodyText) : errBody; } catch { errBody = { error: bodyText || "model not allowed" }; }
+          if (useAuto) {
+            logError(model, 403, errBody.error || "model not allowed");
+            evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
+            lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
+            if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue candidate; }
+            return json(res, 403, errBody);
+          }
           logError(model, 403, errBody.error || "model not allowed");
-          evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true, skipped: true });
-          lastErr = { model, upstream: upRes, status: 403, message: errBody.error || "model not allowed" };
-          if (canFallback && idx < order.length - 1) { evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: `allowlist skip ${errBody.error || "blocked"}` }); continue; }
+          evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
           return json(res, 403, errBody);
         }
-        logError(model, 403, errBody.error || "model not allowed");
-        evt("upstream-error", { reqId, model, status: 403, message: errBody.error, timing: upRes._t ?? null, allowlist: true });
-        return json(res, 403, errBody);
-      }
-      if (auto) await auto.recordError(model, { status: upRes.status });
-      // 读失败响应体（clone 不影响后续 relay 转发原响应；1s 上限防流式错误体拖慢）
-      let upBody = "";
-      try {
-        upBody = String(await Promise.race([
-          upRes.clone().text(),
-          new Promise((r) => { const t = setTimeout(() => r(""), 1000); t.unref?.(); }),
-        ])).replace(/\s+/g, " ").slice(0, 400);
-      } catch {}
-      const upMsg = upBody || `upstream ${upRes.status}`;
-      lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
-      logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}`);
-      evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null });
-      upRes = null;
-    }
-    if (upRes) {
-      const isStream = Boolean(body.stream);
-      const d = hedgeDelayMs();
-      const hasPeers = Boolean(peers) && peers.ordered().length > 0;
-      const canUseGroup = shouldUseGroupForModel(model);
-      const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
-      if (doHedge) {
-        const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res, hedgeDelayMs: d });
-        if (hr.handled) return { done: true };
-        if (hr.lastErr) lastErr = hr.lastErr;
-        if (hr.upRes === null) upRes = null;
-        else if (hr.upRes) upRes = hr.upRes;
+        if (auto) await auto.recordError(model, { status: upRes.status });
+        // 读失败响应体（clone 不影响后续 relay 转发原响应；1s 上限防流式错误体拖慢）
+        let upBody = "";
+        try {
+          upBody = String(await Promise.race([
+            upRes.clone().text(),
+            new Promise((r) => { const t = setTimeout(() => r(""), 1000); t.unref?.(); }),
+          ])).replace(/\s+/g, " ").slice(0, 400);
+        } catch {}
+        const upMsg = upBody || `upstream ${upRes.status}`;
+        lastErr = { model, upstream: upRes, status: upRes.status, message: upMsg };
+        logError(model, upRes.status, `upstream ${upRes.status}${upBody ? ` body=${upBody.slice(0, 300)}` : ""}`);
+        evt("upstream-error", { reqId, model, status: upRes.status, message: upMsg.slice(0, 300), timing: upRes._t ?? null });
+        upRes = null;
+        break;
       }
       if (upRes) {
-        const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res });
-        if (lr.handled) return { done: true };
-        if (lr.lastErr) { lastErr = lr.lastErr; continue; }
-        return { done: true };
+        const isStream = Boolean(body.stream);
+        const d = hedgeDelayMs();
+        const hasPeers = Boolean(peers) && peers.ordered().length > 0;
+        const canUseGroup = shouldUseGroupForModel(model);
+        const doHedge = canUseGroup && shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers, model }) && upRes.status === 200 && upRes.body;
+        if (doHedge) {
+          const hr = await hedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res, hedgeDelayMs: d });
+          if (hr.handled) return { done: true };
+          if (hr.lastErr) lastErr = hr.lastErr;
+          if (hr.upRes === null) upRes = null;
+          else if (hr.upRes) upRes = hr.upRes;
+        }
+        if (upRes) {
+          const lr = await localRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res });
+          if (lr.handled) return { done: true };
+          if (lr.lastErr && isEmptyTurnError(lr.lastErr) && emptyRetried < emptyCfg.max) {
+            emptyRetried++;
+            evt("empty-turn-retry", { reqId, model, retry: emptyRetried, max: emptyCfg.max, delayMs: emptyCfg.delayMs });
+            await sleep(emptyCfg.delayMs);
+            continue;
+          }
+          if (lr.lastErr) { lastErr = lr.lastErr; continue candidate; }
+          return { done: true };
+        }
+        break;
       }
+      break;
     }
     if (canForwardPeers) {
       if (!shouldUseGroupForModel(model)) {

package/src/routes/chat/relay-pipeline.js

@@ -12,6 +12,11 @@ export const LAST_CANDIDATE_TIMEOUT_MS = (() => {
   return Number.isInteger(n) && n >= 0 ? n : 120_000;
 })();
 
+/** 空转 200 判定（serial-trial 同模型重试用）：与 execute 内 _emptyTurn 产生的 lastErr 同源 */
+export function isEmptyTurnError(err) {
+  return Number(err?.status) === 502 && String(err?.message || "").startsWith("EMPTY_MODEL_RESPONSE");
+}
+
 /**
  * RelayPipeline 深模块
  * 把 5 个 handler 各自的 fallback→relay→scoring→事件 6段流水收敛为单一真相。
@@ -132,10 +137,12 @@ export function createRelayPipeline({
       !["tool_calls", "function_call"].includes(_d.sawFinishReason);
     if (_emptyTurn) {
       const _why = _d.sawFinishReason ? ` (finish_reason=${_d.sawFinishReason})` : " (no content, no tool calls)";
-      try { _logError(actual, 502, `empty turn${_why}`); } catch {}
+      // 错误包络暂扣后下游不再直观看到上游原文：把摘要带进最终报错（截断 200 字），排障不断线。
+      const _err = _d.upstreamErrorText ? ` upstream=${String(_d.upstreamErrorText).slice(0, 200)}` : "";
+      try { _logError(actual, 502, `empty turn${_why}${_err}`); } catch {}
       _evt("upstream-error", { reqId, model: actual, status: 502, message: "empty turn", timing: null });
       _evt("fallback", { reqId, from: actual, to: null, reason: "empty turn" });
-      return { handled: false, upRes: null, lastErr: { model: actual, upstream: null, status: 502, message: `EMPTY_MODEL_RESPONSE: upstream returned 200 with no content${_why} — retry or rephrase` } };
+      return { handled: false, upRes: null, lastErr: { model: actual, upstream: null, status: 502, message: `EMPTY_MODEL_RESPONSE: upstream returned 200 with no content${_why}${_err} — retry or rephrase` } };
     }
 
     // 5a. 首块超时未写字节 → 回退（显式 timedOut 字段，status 只是 HTTP 语义展示）

package/src/routes/stream.js

@@ -30,6 +30,40 @@ function hasPayload(text) {
   return false;
 }
 
+// 错误包络 chunk 判定（纯函数）：data 行 JSON 含顶层 .error 对象、或只有 finish_reason=error
+// 的空帧，且整 chunk 无任何 content/tool_calls/reasoning 输出 → 返回错误摘要（建议暂扣），
+// 否则返回 null（透传）。解析失败一律透传（默认保安全）。
+// 背景：网关把上游失败包成 HTTP 200 SSE；暂扣后下游流式 UI 不再展示瞬时错误
+//（如 Vertex 503 + google fallback 400），本轮走空转重试，客户端只看到最终结果。
+function holdableChunk(txt) {
+  if (typeof txt !== "string" || !txt.includes("data:")) return null;
+  let sawData = false;
+  const errs = [];
+  for (const line of txt.split("\n")) {
+    const t = line.trim();
+    if (!t.startsWith("data:")) continue;
+    const d = t.slice(5).trim();
+    if (!d || d === "[DONE]") continue;
+    let j = null;
+    try { j = JSON.parse(d); } catch { return null; }
+    if (!j || typeof j !== "object") return null;
+    sawData = true;
+    const c0 = (Array.isArray(j.choices) && j.choices[0]) || {};
+    const delta = c0.delta || {};
+    const msg = c0.message || {};
+    const out = delta.content || msg.content || delta.tool_calls || msg.tool_calls || delta.reasoning || msg.reasoning;
+    if (out && !(Array.isArray(out) && out.length === 0)) return null;
+    if (j.error && typeof j.error === "object") {
+      const m = j.error.message || j.error.code || "";
+      if (m) errs.push(String(m).slice(0, 300));
+    } else if (c0.finish_reason !== "error") {
+      return null;
+    }
+  }
+  if (!sawData) return null;
+  return errs.length ? errs.join(" | ") : "finish_reason=error 空帧";
+}
+
 // 自持 reader 优先：for-await 会锁定 ReadableStream，使 body.cancel() 必 reject（真流上等于空操作），
 // 超时/断下游要真能掐上游必须走 reader.cancel()
 async function* bodyChunks(body, reader) {
@@ -116,6 +150,8 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
     chars: 0,
     toolCalls: 0,
     chatShaped: false,
+    heldErrorChunks: 0,
+    upstreamErrorText: null,
     recoveries: 0,
   };
   let prevChunkAt = t0;
@@ -206,6 +242,13 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
           if (gap > SCORE_STALL_MS) detail.stallHits += 1;
           prevChunkAt = now;
           const txt = chunkText(chunk);
+          // 错误包络暂扣：本轮尚未写出真实输出时错误帧不写下游（下游 UI 不再展示瞬时错误），
+          // 摘要记 detail.upstreamErrorText 供空转判定与最终报错；已写出真实内容后的错误帧照常透传。
+          const holdErr = !wrotePayload ? holdableChunk(txt) : null;
+          if (holdErr) {
+            detail.heldErrorChunks += 1;
+            if (!detail.upstreamErrorText) detail.upstreamErrorText = String(holdErr).slice(0, 500);
+          }
           const isPayload = hasPayload(txt);
           try {
             if (txt.includes("[DONE]")) detail.sawDone = true;
@@ -252,7 +295,7 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
           // 首块/空闲超时后上游仍吐出了真实数据 → 只是慢，不是死：撤销超时判定，照常转发
           //（cancel 是异步的，竞态窗口内已到达的数据是纯收益；丢掉是纯损失）
           // 注释帧（keepalive）不算：否则对端只要在发心跳，闸门就永远解除
-          if (isPayload && (timedOut || stalled)) {
+          if (isPayload && !holdErr && (timedOut || stalled)) {
             timedOut = false;
             stalled = false;
             detail.recoveries = (detail.recoveries || 0) + 1;
@@ -260,7 +303,7 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
             if (firstTimer) { clearTimeout(firstTimer); firstTimer = null; }
           }
           if (timedOut || stalled || tooLong) break;
-          if (isPayload && first) {
+          if (isPayload && first && !holdErr) {
             first = false;
             ttf = Math.round(now - t0);
             onFirstChunk?.(ttf);
@@ -279,11 +322,13 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
               }
             } catch {}
           }
-          wroteAny = true;
-          if (isPayload) wrotePayload = true;
-          detail.wroteChunks += 1;
-          detail.wroteBytes += Buffer.isBuffer(outChunk) ? outChunk.length : (outChunk?.length ?? len);
-          try { res.write(outChunk); } catch { /* 下游已断开：onClose 已掐上游 */ }
+          if (!holdErr) {
+            wroteAny = true;
+            if (isPayload) wrotePayload = true;
+            detail.wroteChunks += 1;
+            detail.wroteBytes += Buffer.isBuffer(outChunk) ? outChunk.length : (outChunk?.length ?? len);
+            try { res.write(outChunk); } catch { /* 下游已断开：onClose 已掐上游 */ }
+          }
           armStall();
         }
         if (!detail.exitReason) detail.exitReason = detail.downstreamClosed ? "downstream-closed" : "normal";