mslxdff 0.1.29 → 0.1.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mslxdff",
-  "version": "0.1.29",
+  "version": "0.1.31",
   "description": "测试项目，请勿使用。",
   "type": "module",
   "bin": {

package/src/auto.js

@@ -1,4 +1,4 @@
-import { loadModelErrors, saveModelErrors } from "./state.js";
+import { loadModelErrors, saveModelErrors, loadModelLatencies, saveModelLatencies } from "./state.js";
 
 export const DEFAULT_AUTO_MODELS = [
   "deepseek-v4-flash-free",
@@ -50,6 +50,7 @@ export function classifyErrorEvent(evt = {}) {
 
 export const DEFAULT_COOLDOWN_MS = 60_000;
 export const DEFAULT_SLOW_COOLDOWN_MS = 5 * 60_000;
+export const DEFAULT_LATENCY_ALPHA = 0.3;
 
 function effectiveCooldown(entry, slowCooldownMs, cooldownMs) {
   if (entry && entry.slow) return slowCooldownMs || 0;
@@ -63,7 +64,14 @@ function inCooldown(id, errors, now, cooldownMs, slowCooldownMs) {
   return cd > 0 && now - e.at < cd;
 }
 
-export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0, slowCooldownMs = 0 } = {}) {
+// Latency EMA helpers
+function normLatency(e) {
+  if (!e || typeof e !== "object") return null;
+  const ema = Number(e.emaMs);
+  return Number.isFinite(ema) && ema > 0 ? ema : null;
+}
+
+export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0, slowCooldownMs = 0, latencies = {} } = {}) {
   return [...new Set(ids)]
     .filter(Boolean)
     .map((id) => ({
@@ -72,10 +80,12 @@ export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0,
       err: normEntry(errors[id])?.at ?? 0,
       isDeepseek: /deepseek/i.test(id),
       cooling: inCooldown(id, errors, now, cooldownMs, slowCooldownMs),
+      latency: normLatency(latencies[id]) ?? Number.MAX_SAFE_INTEGER,
     }))
     .sort(
       (a, b) =>
         (a.cooling ? 1 : 0) - (b.cooling ? 1 : 0) ||
+        a.latency - b.latency ||
         a.err - b.err ||
         (b.isDeepseek ? 1 : 0) - (a.isDeepseek ? 1 : 0)
     )
@@ -88,10 +98,14 @@ export function createAutoSelector({
   now = () => Date.now(),
   cooldownMs = DEFAULT_COOLDOWN_MS,
   slowCooldownMs = DEFAULT_SLOW_COOLDOWN_MS,
+  latencyAlpha = DEFAULT_LATENCY_ALPHA,
   errors: seedErrors,
+  latencies: seedLatencies,
   persist = (errors, f = file) => saveModelErrors(errors, f ? { file: f } : {}),
+  persistLatencies = (latencies, f = file) => saveModelLatencies(latencies, f ? { file: f } : {}),
 } = {}) {
   const lastErrorAt = { ...(seedErrors ?? loadModelErrors(file ? { file } : {})) };
+  const latencies = { ...(seedLatencies ?? loadModelLatencies(file ? { file } : {})) };
 
   async function loadList() {
     let list;
@@ -105,7 +119,7 @@ export function createAutoSelector({
   }
 
   async function candidates() {
-    return rankModels(await loadList(), lastErrorAt, { now: now(), cooldownMs, slowCooldownMs });
+    return rankModels(await loadList(), lastErrorAt, { now: now(), cooldownMs, slowCooldownMs, latencies });
   }
 
   async function candidatesFor(requested) {
@@ -116,6 +130,7 @@ export function createAutoSelector({
       now: now(),
       cooldownMs,
       slowCooldownMs,
+      latencies,
     });
     if (inCooldown(requested, lastErrorAt, now(), cooldownMs, slowCooldownMs)) {
       return [...others, requested];
@@ -138,23 +153,45 @@ export function createAutoSelector({
     await persist({ ...lastErrorAt });
   }
 
-  async function recordOk(id) {
+  async function recordOk(id, evt = {}) {
     if (!id) return;
     lastErrorAt[id] = { status: MODEL_STATUS.NORMAL, at: now(), code: 200, slow: false };
     await persist({ ...lastErrorAt });
+    const ms = Number(evt.latencyMs ?? evt.totalMs ?? evt.elapsedMs);
+    if (Number.isFinite(ms) && ms > 0) {
+      const prev = latencies[id]?.emaMs;
+      const ema = prev ? Math.round(prev * (1 - latencyAlpha) + ms * latencyAlpha) : Math.round(ms);
+      latencies[id] = { emaMs: ema, lastMs: Math.round(ms), at: now(), count: (latencies[id]?.count ?? 0) + 1 };
+      await persistLatencies({ ...latencies });
+    }
+  }
+
+  async function recordLatency(id, ms) {
+    if (!id || !Number.isFinite(ms) || ms <= 0) return;
+    const prev = latencies[id]?.emaMs;
+    const ema = prev ? Math.round(prev * (1 - latencyAlpha) + ms * latencyAlpha) : Math.round(ms);
+    latencies[id] = { emaMs: ema, lastMs: Math.round(ms), at: now(), count: (latencies[id]?.count ?? 0) + 1 };
+    await persistLatencies({ ...latencies });
   }
 
   function statuses() {
     return { ...lastErrorAt };
   }
 
+  function latencyStatuses() {
+    return { ...latencies };
+  }
+
   return {
     candidates,
     candidatesFor,
     recordError,
     recordOk,
+    recordLatency,
     statuses,
+    latencyStatuses,
     isCooling,
     errors: () => ({ ...lastErrorAt }),
+    latencies: () => ({ ...latencies }),
   };
 }

package/src/routes.js

@@ -65,82 +65,143 @@ function readBody(req) {
   });
 }
 
-// Relay an upstream response to the client. Returns { status, ttfMs, aborted }
-// where:
-//   - status  200 = fully relayed; STREAM_TIMEOUT = first chunk never arrived
-//     within streamTimeoutMs and nothing was written to res yet (safe to
-//     failover); 500 = the response body errored mid-stream.
-//   - ttfMs   time to first chunk when one arrived.
-//   - aborted true when we closed the downstream connection ourselves (only
-//     for the STREAM_TIMEOUT case, before anything was written).
-async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_TIMEOUT_MS } = {}) {
+// Relay an upstream response to the client. Returns { status, ttfMs, aborted, interrupted, detail }
+// detail carries byte/chunk/sawDone diagnostics so a truncated deep-think
+// stream can be told apart from a clean EOF vs our stall/max vs client abort.
+async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort, streamTimeoutMs = STREAM_TIMEOUT_MS } = {}) {
   const t0 = performance.now();
   const contentType = upRes.headers.get("content-type") || "";
   const isStream = Boolean(body?.stream) || contentType.includes("text/event-stream");
   res.statusCode = upRes.status;
 
+  let ttf = null;
+  let interrupted = false;
+  let finishedNormally = false;
+  const detail = {
+    receivedChunks: 0,
+    receivedBytes: 0,
+    wroteChunks: 0,
+    wroteBytes: 0,
+    sawDone: false,
+    sawFinishReason: null,
+    lastChunkAtMs: null,
+    lastChunkGapMs: null,
+    maxGapMs: 0,
+    stallHits: 0, // chunks where gap > SCORE_STALL_MS (quality signal, never cuts)
+    exitReason: null, // normal | first-timeout | stall | max | upstream-error | downstream-close | empty-body
+    upstreamError: null,
+    downstreamClosed: false,
+  };
+  let prevChunkAt = t0;
+  const onClose = () => {
+    detail.downstreamClosed = true;
+    if (!finishedNormally && onDownstreamAbort) onDownstreamAbort();
+  };
+  res.on("close", onClose);
+
   if (isStream) {
     res.setHeader("Content-Type", "text/event-stream");
     res.setHeader("Cache-Control", "no-cache");
     res.setHeader("Connection", "keep-alive");
-    let ttf = null;
-    let interrupted = false;
     if (upRes.body) {
       let first = true;
       let wroteAny = false;
       let timedOut = false;
-      // whole-stream ceiling (TTFB + generation): cancels the upstream body so
-      // the loop exits and we proactively end the stream instead of hanging.
-      let genTooLong = false;
-      const genTimer = GEN_TIMEOUT_MS
-        ? setTimeout(() => {
-            genTooLong = true;
-            if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
-          }, GEN_TIMEOUT_MS)
-        : null;
-      const firstTimer = setTimeout(() => {
+      let stalled = false;
+      let tooLong = false;
+      let stallTimer = null;
+      const armStall = () => {
+        if (stallTimer) clearTimeout(stallTimer);
+        stallTimer = STALL_TIMEOUT_MS
+          ? setTimeout(() => {
+              stalled = true;
+              detail.exitReason = "stall";
+              if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
+            }, STALL_TIMEOUT_MS)
+          : null;
+      };
+      let firstTimer = setTimeout(() => {
         timedOut = true;
+        detail.exitReason = "first-timeout";
         if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
       }, streamTimeoutMs);
+      const maxTimer = MAX_STREAM_MS
+        ? setTimeout(() => {
+            tooLong = true;
+            detail.exitReason = "max";
+            if (typeof upRes.body.cancel === "function") upRes.body.cancel().catch(() => {});
+          }, MAX_STREAM_MS)
+        : null;
       try {
         for await (const chunk of upRes.body) {
-          if (timedOut || genTooLong) break;
+          const now = performance.now();
+          detail.receivedChunks += 1;
+          const len = chunk?.length ?? chunk?.byteLength ?? 0;
+          detail.receivedBytes += len;
+          const gap = Math.round(now - prevChunkAt);
+          detail.lastChunkAtMs = Math.round(now - t0);
+          detail.lastChunkGapMs = gap;
+          if (gap > detail.maxGapMs) detail.maxGapMs = gap;
+          if (gap > SCORE_STALL_MS) detail.stallHits += 1;
+          prevChunkAt = now;
+          // cheap inspection for diagnostics (no full parse)
+          try {
+            const txt = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : typeof chunk === "string" ? chunk : "";
+            if (txt.includes("[DONE]")) detail.sawDone = true;
+            const m = txt.match(/"finish_reason"\s*:\s*"([^"]+)"/);
+            if (m) detail.sawFinishReason = m[1];
+          } catch { /* ignore */ }
+          if (timedOut || stalled || tooLong) break;
           if (first) {
             first = false;
-            ttf = Math.round(performance.now() - t0);
+            ttf = Math.round(now - t0);
             onFirstChunk?.(ttf);
+            if (firstTimer) { clearTimeout(firstTimer); firstTimer = null; }
           }
           wroteAny = true;
+          detail.wroteChunks += 1;
+          detail.wroteBytes += len;
           res.write(chunk);
+          armStall(); // no-op when STALL_TIMEOUT_MS=0; scoring uses SCORE_STALL_MS gap above
         }
+        if (!detail.exitReason) detail.exitReason = "normal";
       } catch (err) {
-        // body threw — if we already started, treat as a mid-stream interrupt
-        if (!wroteAny) { timedOut = true; genTooLong = false; }
+        detail.upstreamError = String(err?.message || err).slice(0, 300);
+        detail.exitReason = "upstream-error";
+        if (!wroteAny) timedOut = true;
+        else stalled = true;
       } finally {
         if (firstTimer) clearTimeout(firstTimer);
-        if (genTimer) clearTimeout(genTimer);
+        if (maxTimer) clearTimeout(maxTimer);
+        if (stallTimer) clearTimeout(stallTimer);
       }
       if (timedOut && !wroteAny) {
-        // nothing written to res yet — safe to drop this model and let the
-        // caller fail over to the next one. Do NOT write/end res here.
-        return { status: STREAM_TIMEOUT_MS, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true };
+        res.removeListener("close", onClose);
+        return { status: STREAM_TIMEOUT_MS, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: true, interrupted: false, detail };
       }
-      if (genTooLong || (timedOut && wroteAny)) {
-        // we'd already started streaming when it exceeded the ceiling — can't
-        // cleanly fail over a half-written response, just end it so the client
-        // sees a clean EOF rather than hanging. interrupted signals the caller
-        // to remember this model as slow.
+      if ((stalled || tooLong) && wroteAny) {
         interrupted = true;
+        detail.exitReason = detail.exitReason || (stalled ? "stall" : "max");
+        res.removeListener("close", onClose);
         try { res.end(); } catch { /* ignore */ }
-        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted };
+        return { status: 200, ttfMs: ttf, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted, detail };
       }
+    } else {
+      detail.exitReason = "empty-body";
     }
     const totalMs = Math.round(performance.now() - t0);
+    if (!detail.exitReason) detail.exitReason = "normal";
+    finishedNormally = true;
+    res.removeListener("close", onClose);
     try { res.end(); } catch { /* ignore */ }
-    return { status: 200, ttfMs: ttf, totalMs, aborted: false };
+    return { status: 200, ttfMs: ttf, totalMs, aborted: false, interrupted: false, detail };
   }
 
+  finishedNormally = true;
+  res.removeListener("close", onClose);
   const text = await upRes.text();
+  detail.receivedBytes = Buffer.byteLength(text);
+  detail.exitReason = "normal-non-stream";
   try {
     json(res, upRes.status, JSON.parse(text));
   } catch {
@@ -148,7 +209,7 @@ async function relay(res, upRes, body, { onFirstChunk, streamTimeoutMs = STREAM_
     res.setHeader("Content-Type", contentType || "text/plain");
     res.end(text);
   }
-  return { status: upRes.status, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: false };
+  return { status: upRes.status, ttfMs: null, totalMs: Math.round(performance.now() - t0), aborted: false, interrupted: false, detail };
 }
 
 const PEER_TIMEOUT_MS = 30_000;
@@ -268,13 +329,25 @@ export const STREAM_TIMEOUT_MS = (() => {
   return Number.isInteger(n) && n > 0 ? n : 25_000;
 })();
 
-// Ceiling on the total wall-clock a single streamed response may run (TTFB +
-// generation + relay). Once exceeded we proactively end the stream so the
-// client gets a clean EOF instead of hanging on a very slow model; the model
-// is then flagged slow and demoted for the next request. Set to 0 to disable.
-export const GEN_TIMEOUT_MS = (() => {
-  const n = Number(process.env.MSLXDFF_GEN_TIMEOUT_MS);
-  return Number.isInteger(n) && n > 0 ? n : 20_000;
+// Stall / max ceilings — disabled by default for relays (we never cut a
+// stream that has already started; different models have different verbosity,
+// that's normal). Stall is kept only as a *quality* signal for ranking.
+// Set MSLXDFF_STALL_TIMEOUT_MS=15000 to re-enable cutting (not recommended),
+// or tune MSLXDFF_SCORE_STALL_MS for scoring.
+export const STALL_TIMEOUT_MS = (() => {
+  const n = Number(process.env.MSLXDFF_STALL_TIMEOUT_MS);
+  return Number.isInteger(n) && n > 0 ? n : 0;
+})();
+
+export const SCORE_STALL_MS = (() => {
+  const raw = process.env.MSLXDFF_SCORE_STALL_MS ?? process.env.MSLXDFF_STALL_TIMEOUT_MS;
+  const n = Number(raw);
+  return Number.isInteger(n) && n > 0 ? n : 15_000;
+})();
+
+export const MAX_STREAM_MS = (() => {
+  const n = Number(process.env.MSLXDFF_MAX_STREAM_MS);
+  return Number.isInteger(n) && n > 0 ? n : 0;
 })();
 
 async function racePeerCandidates(candidates, ctx) {
@@ -355,12 +428,14 @@ const ROUTES = [
       }
 
       const startedAt = Date.now();
+      const reqId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
       const perf0 = performance.now();
       const stages = [];
       const mark = (name) => stages.push([name, Math.round(performance.now() - perf0)]);
       const hops = parseHops(req.headers["x-mslxdff-hops"]);
       const lockModel = req.headers["x-mslxdff-model-lock"] || "";
-      const requested = normalizeModel(lockModel || body.model || "");
+      const rawModel = body.model || "";
+      const requested = normalizeModel(lockModel || rawModel || "");
       const useAuto = isAutoModel(requested);
       mark("parsed");
 
@@ -378,15 +453,16 @@ const ROUTES = [
       mark("ordered");
 
       const logCall = (model, status) =>
-        logs?.appendCall({ model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream), stages });
+        logs?.appendCall({ reqId, model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream), stages });
       const logError = (model, status, message) =>
-        logs?.appendError({ model, auto: useAuto, status, message, stages });
+        logs?.appendError({ reqId, model, auto: useAuto, status, message, stages });
       const evt = (type, data) => {
-        const entry = { ts: Date.now(), type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
+        const entry = { ts: Date.now(), reqId, type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
         if (bus) bus.emit(entry);
         logs?.appendEvent?.(entry);
       };
-      evt("request", { hops, ip: clientIp(req), stream: Boolean(body.stream), prompt: summarizePrompt(body) });
+      evt("request", { reqId, hops, ip: clientIp(req), stream: Boolean(body.stream), prompt: summarizePrompt(body), rawModel, requested, lockModel: lockModel || null });
+      evt("ordered", { reqId, order, canFallback, canForwardPeers, useAuto, statuses: auto?.statuses?.() ?? null });
 
       // Shared context for the peer race helpers below (each model iteration
       // reuses it; `model` is bound per iteration call).
@@ -401,59 +477,87 @@ const ROUTES = [
       };
 
       let lastErr = null;
-      for (const model of order) {
+      for (let idx = 0; idx < order.length; idx++) {
+        const model = order[idx];
         handlerCtx.model = model;
+        evt("model-try", { reqId, model, idx, remaining: order.length - idx });
         let upRes = null;
         const forwarded = { ...injectReasoningContent(model, body), model };
         const tUp = performance.now();
+        evt("upstream-try", { reqId, model, attempt: idx + 1 });
         try {
           upRes = await upstream.chat(forwarded);
+          evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
         } catch (err) {
           if (auto) await auto.recordError(model, { message: errMsg(err) });
           lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
           logError(model, 502, errMsg(err));
-          evt("upstream-error", { model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
+          evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
         }
         mark(`up-${model}`);
         if (upRes && upRes.status >= 400) {
           if (auto) await auto.recordError(model, { status: upRes.status });
           lastErr = { model, upstream: upRes, status: upRes.status, message: null };
           logError(model, upRes.status, `upstream ${upRes.status}`);
-          evt("upstream-error", { model, status: upRes.status, message: null, timing: upRes._t ?? null });
+          evt("upstream-error", { reqId, model, status: upRes.status, message: null, timing: upRes._t ?? null });
           upRes = null;
         }
         if (upRes) {
-          if (auto) await auto.recordOk(model);
           logCall(model, upRes.status);
-          const out = await relay(res, upRes, body, { onFirstChunk: (delta) => mark(`ttf-${model}`) });
+          evt("relay-start", { reqId, model, via: "local", isStream: Boolean(body.stream) });
+          const out = await relay(res, upRes, body, {
+            onFirstChunk: (delta) => {
+              mark(`ttf-${model}`);
+              evt("relay-first-chunk", { reqId, model, ttfMs: delta });
+            },
+            onDownstreamAbort: () => {
+              evt("client-abort", { reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] });
+            },
+          });
+          evt("relay-done", { reqId, model, via: "local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
           if (out.status === STREAM_TIMEOUT_MS) {
-            // nothing was written — treat this model as failed and keep walking
-            // the failover chain instead of waiting out the slow stream.
             if (auto) await auto.recordError(model, { status: 502, slow: true, note: `stream timeout ${STREAM_TIMEOUT_MS}ms` });
             lastErr = { model, upstream: null, status: 502, message: `stream timed out after ${STREAM_TIMEOUT_MS}ms` };
             logError(model, 502, `stream timeout ${STREAM_TIMEOUT_MS}ms`);
-            evt("upstream-error", { model, status: 502, message: "stream timeout", timing: null });
+            evt("upstream-error", { reqId, model, status: 502, message: "stream timeout", timing: null });
+            evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: "stream timeout" });
             upRes = null;
             continue;
           }
           if (out.interrupted) {
-            // the whole stream ran past GEN_TIMEOUT_MS — we ended it proactively
-            // so the client got a clean EOF instead of hanging. Remember this
-            // model as slow so the next request prefers a faster one.
-            if (auto) await auto.recordError(model, { status: 200, slow: true, note: `gen timeout ${GEN_TIMEOUT_MS}ms` });
-            evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: GEN_TIMEOUT_MS, interrupted: true });
+            if (auto) {
+              await auto.recordError(model, { status: 200, slow: true, note: `stall ${STALL_TIMEOUT_MS}ms` });
+              await auto.recordLatency(model, out.totalMs ?? (Date.now() - startedAt));
+            }
+            evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: STALL_TIMEOUT_MS, interrupted: true, detail: out.detail ?? null });
             logCall(model, 200);
-            evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true });
+            evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true, detail: out.detail ?? null });
             return;
           }
-          // A model that took a long wall-clock time (TTFB + generation + relay)
-          // gets remembered as slow so the next request prefers a faster one.
           const elapsed = Date.now() - startedAt;
+          const latencyMs = out.totalMs ?? elapsed;
+          let scoredSlow = false;
           if (SLOW_TOTAL_MS && auto && elapsed > SLOW_TOTAL_MS && out.status === 200) {
             void auto.recordError(model, { status: 200, slow: true, note: `slow ${elapsed}ms` });
-            evt("slow-model", { model, elapsedMs: elapsed, threshold: SLOW_TOTAL_MS });
+            void auto.recordLatency(model, latencyMs);
+            evt("slow-model", { model, elapsedMs: elapsed, threshold: SLOW_TOTAL_MS, reason: "total", detail: out.detail ?? null });
+            scoredSlow = true;
+          }
+          if (out.detail?.stallHits > 0 && auto && out.status === 200) {
+            void auto.recordError(model, { status: 200, slow: true, note: `stall ${out.detail.stallHits}x gap>${SCORE_STALL_MS}ms maxGap ${out.detail.maxGapMs}ms` });
+            void auto.recordLatency(model, latencyMs);
+            evt("slow-model", { model, elapsedMs: elapsed, threshold: SCORE_STALL_MS, reason: "stall", stallHits: out.detail.stallHits, maxGapMs: out.detail.maxGapMs, detail: out.detail ?? null });
+            scoredSlow = true;
           }
-          evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs });
+          if (!scoredSlow && auto && out.status === 200) {
+            await auto.recordOk(model, { latencyMs });
+          } else if (!scoredSlow && auto) {
+            // still update latency for non-200? keep for completeness
+            await auto.recordLatency(model, latencyMs);
+          } else if (scoredSlow && out.detail) {
+            // already recorded slow+latency above, still ensure latency EMA is updated for slow case (done)
+          }
+          evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
           return;
         }
 
@@ -463,36 +567,68 @@ const ROUTES = [
         // once ordered by recovery time (earliest failure first), which
         // favours the peer that has had the longest to come back.
         if (canForwardPeers) {
+          evt("peer-race-start", { reqId, model, peers: peers.ordered().length });
           const win =
             (await racePeerCandidates(peers.ordered(), handlerCtx)) ||
             (await racePeerCandidates(peers.orderedByLastError(), handlerCtx));
           if (win) {
+            evt("peer-race-win", { reqId, model, winPeer: win.peer.url, winTarget: win.target, latencyMs: win.latencyMs });
             await peers.recordResult(win.peer.url, { ok: true, latencyMs: win.latencyMs, model: win.target });
             logCall(win.target, win.res.status);
-            const out = await relay(res, win.res, body, { onFirstChunk: (d) => mark(`ttf-peer-${win.target}`) });
-            evt("result", { model: win.target, status: out.status, via: "peer", timing: win.res._t ?? null, ttfMs: out.ttfMs });
+            evt("relay-start", { reqId, model: win.target, via: "peer", isStream: Boolean(body.stream) });
+            const out = await relay(res, win.res, body, {
+              onFirstChunk: (d) => mark(`ttf-peer-${win.target}`),
+              onDownstreamAbort: () => evt("client-abort", { reqId, model: win.target, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+            });
+            evt("relay-done", { reqId, model: win.target, via: "peer", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+            if (auto && out.status === 200) {
+              const latencyMs = out.totalMs ?? win.latencyMs;
+              if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
+                void auto.recordError(win.target, { status: 200, slow: true, note: `peer slow ${latencyMs}ms` });
+                void auto.recordLatency(win.target, latencyMs);
+              } else {
+                await auto.recordOk(win.target, { latencyMs });
+              }
+            }
+            evt("result", { model: win.target, status: out.status, via: "peer", timing: win.res._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
             return;
           }
+          evt("peer-race-lose", { reqId, model });
         }
 
-        if (canFallback) continue;
+        if (canFallback) {
+          evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` });
+          continue;
+        }
+        evt("exhausted-local", { reqId, lastModel: lastErr?.model ?? model, lastStatus: lastErr?.status ?? 502, order });
         logCall(lastErr?.model ?? model, lastErr?.status ?? 502);
         if (lastErr?.upstream) {
-          const out = await relay(res, lastErr.upstream, body, { onFirstChunk: (d) => mark(`ttf-${lastErr.model}`) });
-          evt("result", { model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs });
+          evt("relay-start", { reqId, model: lastErr.model, via: "local-exhausted", isStream: Boolean(body.stream) });
+          const out = await relay(res, lastErr.upstream, body, {
+            onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
+            onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+          });
+          evt("relay-done", { reqId, model: lastErr.model, via: "local-exhausted", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+          evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
           return;
         }
-        evt("result", { model, status: lastErr?.status ?? 502, via: "none", timing: null });
+        evt("result", { reqId, model, status: lastErr?.status ?? 502, via: "none", timing: null });
         return json(res, 502, { error: lastErr?.message || "all auto models failed" });
       }
 
+      evt("exhausted-all", { reqId, lastModel: lastErr?.model ?? requested, lastStatus: lastErr?.status ?? 502, order });
       logCall(lastErr?.model ?? requested, lastErr?.status ?? 502);
       if (lastErr?.upstream) {
-        const out = await relay(res, lastErr.upstream, body, { onFirstChunk: (d) => mark(`ttf-${lastErr.model}`) });
-        evt("result", { model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs });
+        evt("relay-start", { reqId, model: lastErr.model, via: "local-final", isStream: Boolean(body.stream) });
+        const out = await relay(res, lastErr.upstream, body, {
+          onFirstChunk: (d) => mark(`ttf-${lastErr.model}`),
+          onDownstreamAbort: () => evt("client-abort", { reqId, model: lastErr.model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] }),
+        });
+        evt("relay-done", { reqId, model: lastErr.model, via: "local-final", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+        evt("result", { reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
         return;
       }
-      evt("result", { model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none", timing: null });
+      evt("result", { reqId, model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none", timing: null });
       return json(res, 502, { error: lastErr?.message || "all auto models failed" });
     },
   },

package/src/state.js

@@ -47,6 +47,17 @@ export function saveModelErrors(errors, { file = defaultStateFile() } = {}) {
   return errors;
 }
 
+export function loadModelLatencies({ file = defaultStateFile() } = {}) {
+  const lat = readState(file).modelLatencies;
+  return lat && typeof lat === "object" && !Array.isArray(lat) ? lat : {};
+}
+
+export function saveModelLatencies(latencies, { file = defaultStateFile() } = {}) {
+  const state = readState(file);
+  writeState(file, { ...state, modelLatencies: latencies });
+  return latencies;
+}
+
 export function loadPeers({ file = defaultStateFile() } = {}) {
   const peers = readState(file).peers;
   return Array.isArray(peers) ? peers : [];