mslxdff 0.1.5 → 0.1.6

package/bin/mslxdff.js

@@ -1,19 +1,20 @@
 #!/usr/bin/env node
 import { execFile } from "node:child_process";
-import { readFileSync, existsSync } from "node:fs";
+import { readFileSync, existsSync, statSync, watch, openSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { dirname, join } from "node:path";
 import { startServer, resolvePort } from "../src/server.js";
 import { createRouter } from "../src/routes.js";
 import { createUpstreamClient } from "../src/upstream.js";
 import { createModelsService } from "../src/models.js";
-import { loadToken, refreshToken, setPort, getPort, loadGroupsJoined, saveGroupsJoined } from "../src/state.js";
+import { loadToken, refreshToken, setPort, getPort, loadGroupsJoined, saveGroupsJoined, loadModelErrors } from "../src/state.js";
 import { startDaemon, stopDaemon, writePid, pidFile, logFile, readPid } from "../src/daemon.js";
 import { createAutoSelector } from "../src/auto.js";
 import { createPeersService } from "../src/peers.js";
 import { createGroupsService, createBansService, refreshGroupMembers, syncPeersFromMembers } from "../src/groups.js";
-import { logDir, recentCalls, lastError, appendCall, appendError } from "../src/logs.js";
-const logs = { appendCall, appendError };
+import { logDir, recentCalls, lastError, appendCall, appendError, appendEvent, eventsFile, recentEvents } from "../src/logs.js";
+
+const logs = { appendCall, appendError, appendEvent };
 
 const args = process.argv.slice(2);
 const VERSION = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "package.json"), "utf8")).version;
@@ -78,8 +79,28 @@ if (args.includes("-model") || args.includes("-models")) {
     }
     process.exit(0);
   }
+  if (sub === "status") {
+    const statuses = loadModelErrors();
+    const cacheFile = join(logDir(), "models.json");
+    const cached = readModelsCache(cacheFile);
+    const ids = new Set([
+      ...(cached?.data || []).map((m) => m.id),
+      ...Object.keys(statuses),
+    ]);
+    for (const id of ids) {
+      const e = statuses[id];
+      const st = typeof e === "number" ? "error" : e?.status || "normal";
+      const at = typeof e === "number" ? e : e?.at;
+      const when = at
+        ? `  (${new Date(at).toISOString().slice(5, 19).replace("T", " ")})`
+        : "";
+      const extra = e?.code ? `  HTTP ${e.code}` : "";
+      console.log(`  ${id}  ${st}${when}${extra}`);
+    }
+    process.exit(0);
+  }
   if (sub !== undefined && sub !== "list") {
-    console.error("usage: mslxdff -model list | mslxdff -model refresh");
+    console.error("usage: mslxdff -model list | mslxdff -model status | mslxdff -model refresh");
     process.exit(1);
   }
   const cacheFile = join(logDir(), "models.json");
@@ -109,6 +130,13 @@ if (args.includes("-model") || args.includes("-models")) {
   process.exit(0);
 }
 
+// -debug: live-follow the daemon's event stream (requests, upstream errors,
+// peer forwards, peer errors, results) for investigation
+if (args.includes("-debug") || args.includes("--debug")) {
+  await liveDebug();
+  process.exit(0);
+}
+
 // -creategroup <name> | -group create <name> | -group sync | -group leave <name> | -group list |
 // -addtogroup <leader-host> <name> | -resetban [ip]
 const createGroupArg = argValue("-creategroup", "--creategroup") || groupIs("create", args);
@@ -230,7 +258,7 @@ function markJoined(entry) {
   saveGroupsJoined([...list, entry]);
 }
 
-const errMsg = (err) => String(err?.message || err);
+function errMsg(err) { return String(err?.message || err); }
 
 function readModelsCache(cacheFile) {
   try {
@@ -350,7 +378,7 @@ const auto = createAutoSelector({
     }
   },
 });
-const peers = createPeersService({ cooldownMs: peerCooldownMs() });
+const peers = createPeersService({ cooldownMs: peerCooldownMs(), heatMs: peerHeatMs() });
 const groups = createGroupsService({});
 const bans = createBansService({ windowMs: banWindowMs(), threshold: banThreshold() });
 
@@ -420,6 +448,11 @@ function peerCooldownMs() {
   return Number.isInteger(n) && n > 0 ? n : 30_000;
 }
 
+function peerHeatMs() {
+  const n = Number(process.env.MSLXDFF_PEER_HEAT_MS);
+  return Number.isInteger(n) && n > 0 ? n : 5 * 60_000;
+}
+
 function maxHopsValue() {
   const n = Number(process.env.MSLXDFF_MAX_HOPS);
   return Number.isInteger(n) && n > 0 ? n : 3;
@@ -461,7 +494,9 @@ Usage:
   mslxdff -d                       start as a background daemon
   mslxdff -status                  show current status (daemon, models, recent calls, last error)
   mslxdff -model list              list the free models this proxy serves (cached)
+  mslxdff -model status            show per-model health status (normal/limit/error)
   mslxdff -model refresh           force-refresh the model cache from the upstream
+  mslxdff -debug                   live-follow the daemon event stream (requests, errors, peer forwards)
   mslxdff -stop                    stop the running daemon
   mslxdff -port N                  persist the listen port (restarts the daemon on it if running)
   mslxdff -update                  update mslxdff to the latest published version
@@ -485,6 +520,7 @@ Environment:
   MODELS_REFRESH_MS       model-list background refresh interval (default 7200000)
   MSLXDFF_MODEL_COOLDOWN_MS  fallback cooldown after a model error (default 60000)
   MSLXDFF_PEER_COOLDOWN_MS   peer failover cooldown (default 30000)
+  MSLXDFF_PEER_HEAT_MS       how long a peer success stays hot for fast reuse (default 300000)
   MSLXDFF_GROUP_SYNC_MS   group membership sync interval (default 60000)
   MSLXDFF_MAX_HOPS           max peer-forwarding depth (default 3)
   MSLXDFF_BAN_THRESHOLD   failed joins before an ip is banned (default 5)
@@ -554,8 +590,14 @@ async function printStatus() {
   if (allPeers.length) {
     console.log(`\nfailover targets (${allPeers.length}):`);
     for (const p of allPeers) {
-      const cooling = peers.isCooling(p.url) ? "  [cooling]" : "";
-      console.log(`  ${p.name || p.url}  ${p.url}${cooling}`);
+      const tags = [];
+      if (peers.isCooling(p.url)) tags.push("cooling");
+      if (peers.isHot(p.url)) tags.push("hot");
+      const s = peers.stat(p.url);
+      if (s?.latencyMs != null) tags.push(`${s.latencyMs}ms`);
+      if (s?.fails) tags.push(`${s.fails} fail(s)`);
+      const tag = tags.length ? `  [${tags.join(", ")}]` : "";
+      console.log(`  ${p.name || p.url}  ${p.url}${tag}`);
     }
   }
 
@@ -566,12 +608,16 @@ async function printStatus() {
   }
 
   const modelsFile = join(logDir(), "models.json");
+  const statuses = loadModelErrors();
   if (existsSync(modelsFile)) {
     try {
       const cached = JSON.parse(readFileSync(modelsFile, "utf8"));
       const ids = (cached.data || []).map((m) => m.id).filter(Boolean);
       console.log(`\nmodels (${ids.length} free):`);
-      for (const id of ids) console.log(`  ${id}`);
+      for (const id of ids) {
+        const st = fmtStatus(id, statuses);
+        console.log(`  ${id}${st ? `  [${st}]` : ""}`);
+      }
     } catch {
       console.log("\nmodels: cache unreadable");
     }
@@ -601,8 +647,96 @@ async function printStatus() {
   else console.log(`\nnot running — start with: mslxdff -d`);
 }
 
-function fmtTs(iso) {
-  if (!iso) return "-";
+function fmtStatus(id, statuses) {
+  const e = statuses[id];
+  if (typeof e === "number") return `error ${fmtTs(new Date(e).toISOString())}`;
+  if (!e?.status || e.status === "normal") return "";
+  const when = e.at ? ` ${fmtTs(new Date(e.at).toISOString())}` : "";
+  const code = e.code ? ` HTTP ${e.code}` : "";
+  return `${e.status}${when}${code}`;
+}
+
+function fmtEvent(e) {
+  const t = e?.ts ? new Date(e.ts).toISOString().slice(11, 19) : "--:--:--";
+  const head = `[${t}]`;
+  const m = (x) => x || "-";
+  switch (e?.type) {
+    case "request":
+      return `${head} request       ${m(e.model)}${e.auto ? " (auto)" : ""} hops=${e.hops} from ${e.ip || "?"}${e.stream ? " stream" : ""}`;
+    case "upstream-error":
+      return `${head} upstream err  ${m(e.model)} ${e.status ? `HTTP ${e.status}` : "network"}: ${m(e.message)}`;
+    case "peer-health":
+      return `${head} peer check    ${e.peer} -> ${e.count ? e.healthy.join(", ") : "no healthy models"}`;
+    case "peer-forward":
+      return `${head} forward ->    ${e.peer} model=${m(e.model)} hops=${e.hops}${e.retry ? " (retry)" : ""}`;
+    case "peer-error":
+      return `${head} peer err      ${e.peer} ${e.status ? `HTTP ${e.status}` : "network"}: ${m(e.message)}`;
+    case "result":
+      return `${head} result        ${e.status} ${m(e.model)} via=${e.via} ${e.durationMs ?? "?"}ms`;
+    default:
+      return `${head} ${e?.type || "?"} ${JSON.stringify(e || {})}`;
+  }
+}
+
+// Live-follow the daemon's event stream: print the recent tail, then stream
+// new lines as they are appended (Ctrl+C to exit).
+async function liveDebug() {
+  const file = eventsFile();
+  // ensure the event file exists so watch() doesn't fail on a fresh daemon dir
+  if (!existsSync(file)) {
+    try {
+      openSync(file, "a").close();
+    } catch {
+      console.error(`cannot create event file: ${file}`);
+      process.exit(1);
+    }
+  }
+  const recent = recentEvents(100);
+  if (recent.length) {
+    console.log(`--- last ${recent.length} event(s) ---`);
+    for (const e of recent) console.log(fmtEvent(e));
+  }
+  console.log("--- live (Ctrl+C to exit) ---");
+  if (!existsSync(file)) console.log("(no events yet — trigger a chat request to see the flow)");
+
+  let pos = existsSync(file) ? statSync(file).size : 0;
+  let buf = "";
+  const pump = () => {
+    if (!existsSync(file)) {
+      pos = 0;
+      buf = "";
+      return;
+    }
+    const size = statSync(file).size;
+    if (size < pos) {
+      pos = 0; // file trimmed/rewritten: re-follow from the start of what remains
+      buf = "";
+    }
+    if (size === pos) return;
+    buf += readFileSync(file, "utf8").slice(pos);
+    pos = size;
+    const lines = buf.split("\n");
+    buf = lines.pop() || "";
+    for (const l of lines) {
+      if (!l.trim()) continue;
+      try {
+        console.log(fmtEvent(JSON.parse(l)));
+      } catch {
+        // partial/corrupt line while trimming — skip
+      }
+    }
+  };
+  try {
+    watch(file, { persistent: true }, pump);
+  } catch {
+    console.error(`cannot watch event file: ${file}`);
+    process.exit(1);
+  }
+  pump();
+  await new Promise(() => {}); // run until Ctrl+C
+}
+
+function fmtTs(iso) {  if (!iso) return "-";
   try {
     return new Date(iso).toISOString().replace("T", " ").slice(5, 19);
   } catch {

package/package.json

@@ -1,27 +1,27 @@
-{
-  "name": "mslxdff",
-  "version": "0.1.5",
-  "description": "测试项目，请勿使用。",
-  "type": "module",
-  "bin": {
-    "mslxdff": "bin/mslxdff.js"
-  },
-  "scripts": {
-    "start": "node bin/mslxdff.js",
-    "test": "node --test test/*.test.js"
-  },
-  "engines": {
-    "node": ">=20"
-  },
-  "files": [
-    "bin/",
-    "src/",
-    "README.md"
-  ],
-  "keywords": [
-    "test",
-    "demo",
-    "placeholder"
-  ],
-  "license": "MIT"
-}
+{
+  "name": "mslxdff",
+  "version": "0.1.6",
+  "description": "测试项目，请勿使用。",
+  "type": "module",
+  "bin": {
+    "mslxdff": "bin/mslxdff.js"
+  },
+  "scripts": {
+    "start": "node bin/mslxdff.js",
+    "test": "node --test test/*.test.js"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "README.md"
+  ],
+  "keywords": [
+    "test",
+    "demo",
+    "placeholder"
+  ],
+  "license": "MIT"
+}

package/src/auto.js

@@ -14,12 +14,42 @@ export function isAutoModel(model) {
   return !model || model === "auto";
 }
 
+export const MODEL_STATUS = Object.freeze({
+  NORMAL: "normal",
+  LIMIT: "limit",
+  ERROR: "error",
+});
+
+// Legacy modelErrors entries are bare timestamps ({id: ts}); newer ones are
+// objects ({id: {status, at, code}}). Normalize both to an entry object.
+function normEntry(e) {
+  if (typeof e === "number") return { status: MODEL_STATUS.ERROR, at: e, code: null };
+  if (e && typeof e === "object") {
+    return {
+      status: e.status || MODEL_STATUS.ERROR,
+      at: typeof e.at === "number" ? e.at : 0,
+      code: e.code ?? null,
+    };
+  }
+  return null;
+}
+
+export function classifyErrorEvent(evt = {}) {
+  const code = Number(evt.status);
+  if (code === 429) return MODEL_STATUS.LIMIT;
+  const msg = String(evt.message || evt.note || "").toLowerCase();
+  if (msg.includes("rate limit") || msg.includes("limit exceeded") || msg.includes("429")) {
+    return MODEL_STATUS.LIMIT;
+  }
+  return MODEL_STATUS.ERROR;
+}
+
 export const DEFAULT_COOLDOWN_MS = 60_000;
 
 function inCooldown(id, errors, now, cooldownMs) {
   if (!cooldownMs) return false;
-  const err = errors[id];
-  return typeof err === "number" && now - err < cooldownMs;
+  const at = normEntry(errors[id])?.at ?? 0;
+  return at > 0 && now - at < cooldownMs;
 }
 
 export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0 } = {}) {
@@ -27,7 +57,7 @@ export function rankModels(ids, errors = {}, { now = Date.now(), cooldownMs = 0
     .filter(Boolean)
     .map((id) => ({
       id,
-      err: typeof errors[id] === "number" ? errors[id] : 0,
+      err: normEntry(errors[id])?.at ?? 0,
       isDeepseek: /deepseek/i.test(id),
       cooling: inCooldown(id, errors, now, cooldownMs),
     }))
@@ -79,20 +109,36 @@ export function createAutoSelector({
     return [requested, ...others];
   }
 
-  async function recordError(id) {
+  function isCooling(id) {
+    return inCooldown(id, lastErrorAt, now(), cooldownMs);
+  }
+
+  async function recordError(id, evt = {}) {
     if (!id) return;
-    lastErrorAt[id] = now();
+    lastErrorAt[id] = {
+      status: classifyErrorEvent(evt),
+      at: now(),
+      code: Number.isInteger(Number(evt.status)) ? Number(evt.status) : null,
+    };
     await persist({ ...lastErrorAt });
   }
 
-  function isCooling(id) {
-    return inCooldown(id, lastErrorAt, now(), cooldownMs);
+  async function recordOk(id) {
+    if (!id) return;
+    lastErrorAt[id] = { status: MODEL_STATUS.NORMAL, at: now(), code: 200 };
+    await persist({ ...lastErrorAt });
+  }
+
+  function statuses() {
+    return { ...lastErrorAt };
   }
 
   return {
     candidates,
     candidatesFor,
     recordError,
+    recordOk,
+    statuses,
     isCooling,
     errors: () => ({ ...lastErrorAt }),
   };

package/src/logs.js

@@ -19,6 +19,10 @@ export function errorsFile() {
   return join(logDir(), "errors.log");
 }
 
+export function eventsFile() {
+  return join(logDir(), "events.log");
+}
+
 function ensureDir(dir) {
   mkdirSync(dir, { recursive: true });
 }
@@ -50,6 +54,16 @@ export function appendError(entry, { file = errorsFile() } = {}) {
   appendLine(file, entry);
 }
 
+// Structured debug event stream: one JSON line per request/error/forward
+// step, consumed live by `mslxdff -debug`.
+export function appendEvent(entry, { file = eventsFile() } = {}) {
+  appendLine(file, entry);
+}
+
+export function recentEvents(n = 100, { file = eventsFile() } = {}) {
+  return readLines(file).slice(-n);
+}
+
 function readLines(file) {
   try {
     if (!existsSync(file)) return [];

package/src/peers.js

@@ -1,8 +1,11 @@
-import { loadPeers, savePeers, loadPeerErrors, savePeerErrors } from "./state.js";
+import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
 
 export const DEFAULT_PEER_COOLDOWN_MS = 30_000;
+export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
 export const DEFAULT_MAX_HOPS = 3;
 
+const EMA_ALPHA = 0.3;
+
 export function normalizePeerUrl(url) {
   return String(url || "").trim().replace(/\/+$/, "");
 }
@@ -11,15 +14,19 @@ export function createPeersService({
   file,
   now = () => Date.now(),
   cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
+  heatMs = DEFAULT_PEER_HEAT_MS,
   peers: seedPeers,
   errors: seedErrors,
+  stats: seedStats,
   persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
   persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
+  persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
 } = {}) {
   const list = (seedPeers ?? loadPeers(file ? { file } : {}))
     .map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
     .filter((p) => p && p.url);
   const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
+  const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
 
   function all() {
     return [...list];
@@ -64,6 +71,32 @@ export function createPeersService({
     return list.filter((p) => !isCooling(p.url));
   }
 
+  // A peer counts as "hot" when it succeeded recently: reuse its last model
+  // without a fresh health probe to keep the fast path fast.
+  function isHot(url, t = now()) {
+    if (isCooling(url)) return false;
+    const s = stats[url];
+    return Boolean(s?.okAt) && t - s.okAt < heatMs;
+  }
+
+  function stat(url) {
+    return stats[url] ? { ...stats[url] } : null;
+  }
+
+  function rankScore(p, t) {
+    const s = stats[p.url];
+    const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
+    const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
+    const fails = s?.fails ?? 0;
+    return hot * 1_000_000_000 + latency * 1_000 + fails;
+  }
+
+  // Available peers ordered for failover: hot (recent success, low latency,
+  // few failures) first, cold/unused peers last.
+  function ordered(t = now()) {
+    return available().sort((a, b) => rankScore(a, t) - rankScore(b, t));
+  }
+
   let cursor = 0;
 
   function next() {
@@ -79,5 +112,26 @@ export function createPeersService({
     await persistErrors({ ...lastErrorAt });
   }
 
-  return { all, add, remove, removeByGroup, isCooling, available, next, recordError, errors: () => ({ ...lastErrorAt }) };
+  // Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
+  // last successful model); failures bump the consecutive-failure counter.
+  async function recordResult(url, { ok, latencyMs, model } = {}) {
+    if (!url) return;
+    if (ok) {
+      const prev = stats[url] || {};
+      stats[url] = {
+        okAt: now(),
+        latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
+          ? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
+          : (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
+        fails: 0,
+        model: model || prev.model || "",
+      };
+    } else {
+      const prev = stats[url] || {};
+      stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
+    }
+    await persistStats({ ...stats });
+  }
+
+  return { all, add, remove, removeByGroup, isCooling, isHot, stat, ordered, available, next, recordError, recordResult, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }) };
 }

package/src/routes.js

@@ -1,4 +1,5 @@
 import { timingSafeEqual, createHash } from "node:crypto";
+import { performance } from "node:perf_hooks";
 import { injectReasoningContent, normalizeModel } from "./reasoning.js";
 import { isAutoModel } from "./auto.js";
 import { DEFAULT_MAX_HOPS } from "./peers.js";
@@ -92,6 +93,29 @@ async function relay(res, upRes, body) {
 }
 
 const PEER_TIMEOUT_MS = 30_000;
+const PEER_STATUS_TIMEOUT_MS = 2_000;
+
+// Ask a peer which of its models are healthy (status normal or never
+// failed). Returns model ids ordered as the peer listed them; empty when the
+// peer is unreachable, unauthorized, or has no healthy model.
+export async function peerHealthyModels(peer, { timeoutMs = PEER_STATUS_TIMEOUT_MS, fetchImpl = fetch } = {}) {
+  try {
+    const res = await fetchImpl(`${peer.url}/v1/models/status`, {
+      headers: {
+        "Authorization": `Bearer ${peer.token || ""}`,
+        "Accept": "application/json",
+      },
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!res.ok) return [];
+    const json = await res.json().catch(() => ({}));
+    return (json.data || [])
+      .filter((m) => m && typeof m.id === "string" && m.status === "normal")
+      .map((m) => m.id);
+  } catch {
+    return [];
+  }
+}
 
 function parseHops(header) {
   const n = Number(header);
@@ -161,6 +185,8 @@ const ROUTES = [
         logs?.appendCall({ model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream) });
       const logError = (model, status, message) =>
         logs?.appendError({ model, auto: useAuto, status, message });
+      const evt = (type, data) => logs?.appendEvent?.({ type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt });
+      evt("request", { hops, ip: clientIp(req), stream: Boolean(body.stream) });
 
       let lastErr = null;
       for (const model of order) {
@@ -169,47 +195,100 @@ const ROUTES = [
         try {
           upRes = await upstream.chat(forwarded);
         } catch (err) {
-          if (auto) await auto.recordError(model);
+          if (auto) await auto.recordError(model, { message: errMsg(err) });
           lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
           logError(model, 502, errMsg(err));
+          evt("upstream-error", { model, status: 502, message: errMsg(err) });
         }
         if (upRes && upRes.status >= 400) {
-          if (auto) await auto.recordError(model);
+          if (auto) await auto.recordError(model, { status: upRes.status });
           lastErr = { model, upstream: upRes, status: upRes.status, message: null };
           logError(model, upRes.status, `upstream ${upRes.status}`);
+          evt("upstream-error", { model, status: upRes.status, message: null });
           upRes = null;
         }
         if (upRes) {
+          if (auto) await auto.recordOk(model);
           logCall(model, upRes.status);
+          evt("result", { model, status: upRes.status, via: "local" });
           return relay(res, upRes, body);
         }
 
-        // local failed for this model: try the same model on peers, round-robin over available ones
+        // local failed for this model: try peers ordered hot-first (reuse
+        // their last successful model without probing), cold peers get a
+        // health probe before the forward
         if (canForwardPeers) {
-          const count = peers.available().length;
-          for (let i = 0; i < count; i++) {
-            const peer = peers.next();
-            if (!peer) break;
-            let peerRes = await forwardToPeer(peer, body, model, hops);
+          for (const peer of peers.ordered()) {
+            let target = null;
+            const hot = peers.isHot(peer.url);
+            if (hot && peers.stat(peer.url)?.model) {
+              target = peers.stat(peer.url).model;
+            } else {
+              const healthy = await peerHealthyModels(peer);
+              if (!healthy.length) {
+                // peer unreachable or every model unhealthy — mark it and move on
+                await peers.recordError(peer.url);
+                logError(model, 0, `peer ${peer.url} has no healthy models`);
+                evt("peer-health", { peer: peer.url, healthy: [], count: 0 });
+                continue;
+              }
+              evt("peer-health", { peer: peer.url, healthy, count: healthy.length });
+              target = healthy[0];
+            }
+
+            const t0 = performance.now();
+            let peerRes = await forwardToPeer(peer, body, target, hops);
+            let latencyMs = Math.round(performance.now() - t0);
+            evt("peer-forward", { peer: peer.url, model: target, hops: hops + 1 });
+            if (peerRes instanceof Error || peerRes.status >= 400) {
+              // hot-cache miss: probe this peer once for a healthy model
+              if (hot) {
+                const healthy = await peerHealthyModels(peer);
+                if (healthy.length) {
+                  target = healthy[0];
+                  const t1 = performance.now();
+                  peerRes = await forwardToPeer(peer, body, target, hops);
+                  latencyMs = Math.round(performance.now() - t1);
+                  evt("peer-forward", { peer: peer.url, model: target, hops: hops + 1, retry: true });
+                }
+              }
+            }
             if (peerRes instanceof Error || peerRes.status >= 400) {
               await peers.recordError(peer.url);
+              await peers.recordResult(peer.url, { ok: false });
               logError(model, peerRes instanceof Error ? 502 : peerRes.status,
                 peerRes instanceof Error ? errMsg(peerRes) : `peer ${peerRes.status}`);
+              evt("peer-error", {
+                peer: peer.url,
+                model: target,
+                status: peerRes instanceof Error ? 502 : peerRes.status,
+                message: peerRes instanceof Error ? errMsg(peerRes) : null,
+              });
               continue;
             }
-            logCall(model, peerRes.status);
+            await peers.recordResult(peer.url, { ok: true, latencyMs, model: target });
+            logCall(target, peerRes.status);
+            evt("result", { model: target, status: peerRes.status, via: "peer" });
             return relay(res, peerRes, body);
           }
         }
 
         if (canFallback) continue;
         logCall(lastErr?.model ?? model, lastErr?.status ?? 502);
-        if (lastErr?.upstream) return relay(res, lastErr.upstream, body);
+        if (lastErr?.upstream) {
+          evt("result", { model: lastErr.model, status: lastErr.status, via: "local" });
+          return relay(res, lastErr.upstream, body);
+        }
+        evt("result", { model, status: lastErr?.status ?? 502, via: "none" });
         return json(res, 502, { error: lastErr?.message || "all auto models failed" });
       }
 
       logCall(lastErr?.model ?? requested, lastErr?.status ?? 502);
-      if (lastErr?.upstream) return relay(res, lastErr.upstream, body);
+      if (lastErr?.upstream) {
+        evt("result", { model: lastErr.model, status: lastErr.status, via: "local" });
+        return relay(res, lastErr.upstream, body);
+      }
+      evt("result", { model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none" });
       return json(res, 502, { error: lastErr?.message || "all auto models failed" });
     },
   },
@@ -306,4 +385,30 @@ const ROUTES = [
       }
     },
   },
+  {
+    method: "GET",
+    path: "/v1/models/status",
+    requiresAuth: true,
+    handler: async ({ res, models, auto }) => {
+      const statuses = auto?.statuses?.() || {};
+      let ids = [];
+      try {
+        ids = (await models?.get?.())?.data?.map((m) => m.id) || [];
+      } catch {
+        // models list unavailable; fall back to status records only
+      }
+      const seen = new Set();
+      const data = [];
+      for (const id of [...ids, ...Object.keys(statuses)]) {
+        if (seen.has(id)) continue;
+        seen.add(id);
+        const e = statuses[id];
+        const entry = typeof e === "number"
+          ? { id, status: "error", at: e }
+          : { id, status: e?.status || "normal", at: e?.at ?? null, code: e?.code ?? null };
+        data.push(entry);
+      }
+      json(res, 200, { object: "list", data });
+    },
+  },
 ];

package/src/state.js

@@ -69,6 +69,17 @@ export function savePeerErrors(errors, { file = defaultStateFile() } = {}) {
   return errors;
 }
 
+export function loadPeerStats({ file = defaultStateFile() } = {}) {
+  const stats = readState(file).peerStats;
+  return stats && typeof stats === "object" && !Array.isArray(stats) ? stats : {};
+}
+
+export function savePeerStats(stats, { file = defaultStateFile() } = {}) {
+  const state = readState(file);
+  writeState(file, { ...state, peerStats: stats });
+  return stats;
+}
+
 export function loadGroups({ file = defaultStateFile() } = {}) {
   const groups = readState(file).groups;
   return groups && typeof groups === "object" && !Array.isArray(groups) ? groups : {};