mslxdff 0.1.199 → 0.1.203

package/src/peers.js

@@ -1,251 +1,254 @@
-import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
-
-export const DEFAULT_PEER_COOLDOWN_MS = 30_000;
-export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 5 * 60_000;
-export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
-export const DEFAULT_MAX_HOPS = 3;
-export const DEFAULT_BROADBAND_STALE_MS = 90_000;
-// 连续几次连接级失败即视为"这个地址没服务"，进长冷却（不再靠 30s 短冷却反复捞回）。
-export const DEAD_FAIL_THRESHOLD = 2;
-
-const EMA_ALPHA = 0.3;
-
-export function broadbandStaleMs() {
-  const n = Number(process.env.MSLXDFF_BROADBAND_STALE_MS);
-  return Number.isInteger(n) && n > 0 ? n : DEFAULT_BROADBAND_STALE_MS;
-}
-
-export function isBroadbandStale(peer, nowMs = Date.now(), staleMs = broadbandStaleMs()) {
-  const kind = peer?.kind || (String(peer?.url || "").startsWith("relay://") ? "broadband" : "static");
-  if (kind !== "broadband") return false;
-  const lastSeen = peer?.lastSeen;
-  if (typeof lastSeen !== "number") return true; // no heartbeat yet => stale
-  return nowMs - lastSeen > staleMs;
-}
-
-export function normalizePeerUrl(url) {
-  return String(url || "").trim().replace(/\/+$/, "");
-}
-
-export function createPeersService({
-  file,
-  now = () => Date.now(),
-  cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
-  limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
-  heatMs = DEFAULT_PEER_HEAT_MS,
-  peers: seedPeers,
-  errors: seedErrors,
-  stats: seedStats,
-  persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
-  persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
-  persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
-} = {}) {
-  const list = (seedPeers ?? loadPeers(file ? { file } : {}))
-    .map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
-    .filter((p) => p && p.url);
-  const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
-  const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
-
-  function all() {
-    return [...list];
-  }
-
-  function add(peer) {
-    const url = normalizePeerUrl(peer?.url);
-    if (!url) return false;
-    const existing = list.find((p) => p.url === url);
-    const entry = { ...peer, url };
-    if (existing) Object.assign(existing, entry);
-    else list.push(entry);
-    persistPeers([...list]);
-    return true;
-  }
-
-  function remove(url) {
-    const target = normalizePeerUrl(url);
-    const idx = list.findIndex((p) => p.url === target);
-    if (idx < 0) return false;
-    list.splice(idx, 1);
-    persistPeers([...list]);
-    return true;
-  }
-
-  function removeByGroup(group) {
-    const before = list.length;
-    for (let i = list.length - 1; i >= 0; i--) {
-      if (list[i].group === group) list.splice(i, 1);
-    }
-    if (list.length !== before) persistPeers([...list]);
-    return before - list.length;
-  }
-
-  // 分级冷却：连续 429（streak >= 2）→ limitCooldownMs（限流恢复慢，5min 内不再考虑）；
-  // 连接级永久失败（ECONNREFUSED/ENOTFOUND，deadFails >= 2）同样进长冷却；
-  // 其他失败（网络抖动、5xx）→ cooldownMs（可能几秒恢复，给快速重试机会）。
-  function coolingWindowMs(url) {
-    if ((stats[url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return limitCooldownMs;
-    const streak = stats[url]?.streak429 ?? 0;
-    return streak >= 2 ? limitCooldownMs : cooldownMs;
-  }
-
-  // 连接级失败（端口没服务/DNS 解析不了）与"这一发不顺"是两回事：前者换多少次都一样，
-  // 现网 node-b（127.0.0.1:3500，本机根本没服务）靠 30s 短冷却 + 兜底，被反复竞速 1569 次。
-  function isConnLevelFailure(message) {
-    const s = String(message || "");
-    return /ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ECONNRESET|fetch failed|connect (?:timeout|ETIMEDOUT)|socket hang up/i.test(s);
-  }
-
-  function isCooling(url) {
-    const win = coolingWindowMs(url);
-    if (!win) return false;
-    const err = lastErrorAt[url];
-    if (typeof err === "number" && now() - err < win) return true;
-    const peer = list.find((p) => p.url === url);
-    if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
-    return false;
-  }
-
-  function isBroadbandCooling(url) {
-    const peer = list.find((p) => p.url === url);
-    if (!peer) return false;
-    return isBroadbandStale(peer, now(), broadbandStaleMs());
-  }
-
-  function available() {
-    return list.filter((p) => !isCooling(p.url));
-  }
-
-  // A peer counts as "hot" when it succeeded recently: reuse its last model
-  // without a fresh health probe to keep the fast path fast.
-  function isHot(url, t = now()) {
-    if (isCooling(url)) return false;
-    const s = stats[url];
-    return Boolean(s?.okAt) && t - s.okAt < heatMs;
-  }
-
-  function stat(url) {
-    return stats[url] ? { ...stats[url] } : null;
-  }
-
-  function rankScore(p, t) {
-    const s = stats[p.url];
-    const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
-    const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
-    const fails = s?.fails ?? 0;
-    return hot * 1_000_000_000 + latency * 1_000 + fails;
-  }
-
-  // Available peers ordered for failover: hot peers (recent success) first —
-  // the winner of the last race is reused next. Everyone else is ordered by
-  // when they last failed, earliest first: a peer that failed longer ago has
-  // had more time to recover, so it is tried before a more recent failure.
-  // Peers with no recorded error go after any that failed.
-  function ordered(t = now()) {
-    return available().sort((a, b) => {
-      const hotA = stats[a.url]?.okAt && t - stats[a.url].okAt < heatMs ? 0 : 1;
-      const hotB = stats[b.url]?.okAt && t - stats[b.url].okAt < heatMs ? 0 : 1;
-      if (hotA !== hotB) return hotA - hotB;
-      const ea = lastErrorAt[a.url];
-      const eb = lastErrorAt[b.url];
-      if (ea == null && eb == null) return 0;
-      if (ea == null) return 1; // never-failed peers go after any that failed
-      if (eb == null) return -1;
-      return ea - eb; // earliest failure first
-    });
-  }
-
-  // Available peers ordered specifically for recovery-time retry: earliest
-  // failure first (it has had the longest to come back), never-failed last.
-  function orderedByLastError(t = now()) {
-    return available().sort((a, b) => {
-      const ea = lastErrorAt[a.url];
-      const eb = lastErrorAt[b.url];
-      if (ea == null && eb == null) return 0;
-      if (ea == null) return 1; // never-failed peers go after any that failed
-      if (eb == null) return -1;
-      return ea - eb; // earliest failure first
-    });
-  }
-
-  let cursor = 0;
-
-  // 兜底重试集合：仅"因错误冷却中"的 peer（按最早失败优先）。
-  // 场景：唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时，兜底给它一次机会；
-  // 成功会被 recordResult 清除错误记录、回归热路径，避免"明明能用的节点被 30s 冷却全灭"。
-  function coolingByLastError() {
-    const t = now();
-    return list
-      .filter((p) => {
-        const err = lastErrorAt[p.url];
-        if (typeof err !== "number") return false;
-        if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却：兜底也不考虑
-        if ((stats[p.url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return false; // 连接级永久失败：兜底也不考虑（否则每次白等一次拒连）
-        return t - err < cooldownMs;
-      })
-      .sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
-  }
-
-  function next() {
-    const avail = available();
-    if (!avail.length) return null;
-    cursor = cursor % avail.length;
-    return avail[cursor++];
-  }
-
-  // Long-lived error memory: a peer keeps its last-error timestamp until a
-  // subsequent success resets it (success clears the failure record) or the
-  // error is no longer in the persist store on next load.
-  async function recordError(url, { status, message } = {}) {
-    if (!url) return;
-    lastErrorAt[url] = now();
-    await persistErrors({ ...lastErrorAt });
-    // 连续 429 计数（仅成功清零）：第 2 次起进入长冷却——该 peer 大概率真被限流，不再频繁试它
-    if (Number(status) === 429) {
-      const prev = stats[url] || {};
-      stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
-      await persistStats({ ...stats });
-    }
-    // 连接级失败计数（仅成功清零）：第 2 次起进长冷却，且不再被 coolingByLastError 兜底捞回
-    if (isConnLevelFailure(message)) {
-      const prev = stats[url] || {};
-      stats[url] = { ...prev, deadFails: (prev.deadFails || 0) + 1 };
-      await persistStats({ ...stats });
-    }
-  }
-
-  // Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
-  // last successful model) and clears the error memory; failures keep the
-  // error timestamp so the retry pass can order by recovery time.
-  async function recordResult(url, { ok, latencyMs, model } = {}) {
-    if (!url) return;
-    if (ok) {
-      if (lastErrorAt[url] != null) {
-        delete lastErrorAt[url];
-        await persistErrors({ ...lastErrorAt });
-      }
-      const prev = stats[url] || {};
-      stats[url] = {
-        okAt: now(),
-        latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
-          ? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
-          : (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
-        fails: 0,
-        model: model || prev.model || "",
-        streak429: 0,
-        deadFails: 0,
-      };
-    } else {
-      const prev = stats[url] || {};
-      stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
-    }
-    await persistStats({ ...stats });
-  }
-
-  return {
-    all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
-      const peer = list.find((p) => p.url === url);
-      return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
-    }, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
-    recordError, recordResult, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
-  };
-}
+import { loadPeers, savePeers, loadPeerErrors, savePeerErrors, loadPeerStats, savePeerStats } from "./state.js";
+
+// 组员失败冷却分级（2026-10-18 用户拍板）：
+// 非额度失败（502/5xx/超时，可能几秒就恢复）→ 短冷却 10s，到点自动重新纳入候选、继续挨个重试；
+// 额度类（连续 2 次 429）→ 长冷却 10min（额度当天不会因快重试恢复，反复撞只会白烧配额）。
+export const DEFAULT_PEER_COOLDOWN_MS = 10_000;
+export const DEFAULT_PEER_LIMIT_COOLDOWN_MS = 10 * 60_000;
+export const DEFAULT_PEER_HEAT_MS = 5 * 60_000;
+export const DEFAULT_MAX_HOPS = 3;
+export const DEFAULT_BROADBAND_STALE_MS = 90_000;
+// 连续几次连接级失败即视为"这个地址没服务"，进长冷却（不再靠 30s 短冷却反复捞回）。
+export const DEAD_FAIL_THRESHOLD = 2;
+
+const EMA_ALPHA = 0.3;
+
+export function broadbandStaleMs() {
+  const n = Number(process.env.MSLXDFF_BROADBAND_STALE_MS);
+  return Number.isInteger(n) && n > 0 ? n : DEFAULT_BROADBAND_STALE_MS;
+}
+
+export function isBroadbandStale(peer, nowMs = Date.now(), staleMs = broadbandStaleMs()) {
+  const kind = peer?.kind || (String(peer?.url || "").startsWith("relay://") ? "broadband" : "static");
+  if (kind !== "broadband") return false;
+  const lastSeen = peer?.lastSeen;
+  if (typeof lastSeen !== "number") return true; // no heartbeat yet => stale
+  return nowMs - lastSeen > staleMs;
+}
+
+export function normalizePeerUrl(url) {
+  return String(url || "").trim().replace(/\/+$/, "");
+}
+
+export function createPeersService({
+  file,
+  now = () => Date.now(),
+  cooldownMs = DEFAULT_PEER_COOLDOWN_MS,
+  limitCooldownMs = DEFAULT_PEER_LIMIT_COOLDOWN_MS,
+  heatMs = DEFAULT_PEER_HEAT_MS,
+  peers: seedPeers,
+  errors: seedErrors,
+  stats: seedStats,
+  persistPeers = (list, f = file) => savePeers(list, f ? { file: f } : {}),
+  persistErrors = (errors, f = file) => savePeerErrors(errors, f ? { file: f } : {}),
+  persistStats = (stats, f = file) => savePeerStats(stats, f ? { file: f } : {}),
+} = {}) {
+  const list = (seedPeers ?? loadPeers(file ? { file } : {}))
+    .map((p) => ({ ...p, url: normalizePeerUrl(p.url) }))
+    .filter((p) => p && p.url);
+  const lastErrorAt = { ...(seedErrors ?? loadPeerErrors(file ? { file } : {})) };
+  const stats = { ...(seedStats ?? loadPeerStats(file ? { file } : {})) };
+
+  function all() {
+    return [...list];
+  }
+
+  function add(peer) {
+    const url = normalizePeerUrl(peer?.url);
+    if (!url) return false;
+    const existing = list.find((p) => p.url === url);
+    const entry = { ...peer, url };
+    if (existing) Object.assign(existing, entry);
+    else list.push(entry);
+    persistPeers([...list]);
+    return true;
+  }
+
+  function remove(url) {
+    const target = normalizePeerUrl(url);
+    const idx = list.findIndex((p) => p.url === target);
+    if (idx < 0) return false;
+    list.splice(idx, 1);
+    persistPeers([...list]);
+    return true;
+  }
+
+  function removeByGroup(group) {
+    const before = list.length;
+    for (let i = list.length - 1; i >= 0; i--) {
+      if (list[i].group === group) list.splice(i, 1);
+    }
+    if (list.length !== before) persistPeers([...list]);
+    return before - list.length;
+  }
+
+  // 分级冷却：连续 429（streak >= 2）→ limitCooldownMs（限流恢复慢，5min 内不再考虑）；
+  // 连接级永久失败（ECONNREFUSED/ENOTFOUND，deadFails >= 2）同样进长冷却；
+  // 其他失败（网络抖动、5xx）→ cooldownMs（可能几秒恢复，给快速重试机会）。
+  function coolingWindowMs(url) {
+    if ((stats[url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return limitCooldownMs;
+    const streak = stats[url]?.streak429 ?? 0;
+    return streak >= 2 ? limitCooldownMs : cooldownMs;
+  }
+
+  // 连接级失败（端口没服务/DNS 解析不了）与"这一发不顺"是两回事：前者换多少次都一样，
+  // 现网 node-b（127.0.0.1:3500，本机根本没服务）靠 30s 短冷却 + 兜底，被反复竞速 1569 次。
+  function isConnLevelFailure(message) {
+    const s = String(message || "");
+    return /ECONNREFUSED|ENOTFOUND|ENETUNREACH|EHOSTUNREACH|ECONNRESET|fetch failed|connect (?:timeout|ETIMEDOUT)|socket hang up/i.test(s);
+  }
+
+  function isCooling(url) {
+    const win = coolingWindowMs(url);
+    if (!win) return false;
+    const err = lastErrorAt[url];
+    if (typeof err === "number" && now() - err < win) return true;
+    const peer = list.find((p) => p.url === url);
+    if (peer && isBroadbandStale(peer, now(), broadbandStaleMs())) return true;
+    return false;
+  }
+
+  function isBroadbandCooling(url) {
+    const peer = list.find((p) => p.url === url);
+    if (!peer) return false;
+    return isBroadbandStale(peer, now(), broadbandStaleMs());
+  }
+
+  function available() {
+    return list.filter((p) => !isCooling(p.url));
+  }
+
+  // A peer counts as "hot" when it succeeded recently: reuse its last model
+  // without a fresh health probe to keep the fast path fast.
+  function isHot(url, t = now()) {
+    if (isCooling(url)) return false;
+    const s = stats[url];
+    return Boolean(s?.okAt) && t - s.okAt < heatMs;
+  }
+
+  function stat(url) {
+    return stats[url] ? { ...stats[url] } : null;
+  }
+
+  function rankScore(p, t) {
+    const s = stats[p.url];
+    const hot = s?.okAt && t - s.okAt < heatMs ? 0 : 1;
+    const latency = s?.latencyMs != null ? s.latencyMs : 1_000_000;
+    const fails = s?.fails ?? 0;
+    return hot * 1_000_000_000 + latency * 1_000 + fails;
+  }
+
+  // Available peers ordered for failover: hot peers (recent success) first —
+  // the winner of the last race is reused next. Everyone else is ordered by
+  // when they last failed, earliest first: a peer that failed longer ago has
+  // had more time to recover, so it is tried before a more recent failure.
+  // Peers with no recorded error go after any that failed.
+  function ordered(t = now()) {
+    return available().sort((a, b) => {
+      const hotA = stats[a.url]?.okAt && t - stats[a.url].okAt < heatMs ? 0 : 1;
+      const hotB = stats[b.url]?.okAt && t - stats[b.url].okAt < heatMs ? 0 : 1;
+      if (hotA !== hotB) return hotA - hotB;
+      const ea = lastErrorAt[a.url];
+      const eb = lastErrorAt[b.url];
+      if (ea == null && eb == null) return 0;
+      if (ea == null) return 1; // never-failed peers go after any that failed
+      if (eb == null) return -1;
+      return ea - eb; // earliest failure first
+    });
+  }
+
+  // Available peers ordered specifically for recovery-time retry: earliest
+  // failure first (it has had the longest to come back), never-failed last.
+  function orderedByLastError(t = now()) {
+    return available().sort((a, b) => {
+      const ea = lastErrorAt[a.url];
+      const eb = lastErrorAt[b.url];
+      if (ea == null && eb == null) return 0;
+      if (ea == null) return 1; // never-failed peers go after any that failed
+      if (eb == null) return -1;
+      return ea - eb; // earliest failure first
+    });
+  }
+
+  let cursor = 0;
+
+  // 兜底重试集合：仅"因错误冷却中"的 peer（按最早失败优先）。
+  // 场景：唯一可用节点偶发失败被冷却锁死 → 主链零候选/全败时，兜底给它一次机会；
+  // 成功会被 recordResult 清除错误记录、回归热路径，避免"明明能用的节点被 30s 冷却全灭"。
+  function coolingByLastError() {
+    const t = now();
+    return list
+      .filter((p) => {
+        const err = lastErrorAt[p.url];
+        if (typeof err !== "number") return false;
+        if ((stats[p.url]?.streak429 ?? 0) >= 2) return false; // 429 长冷却：兜底也不考虑
+        if ((stats[p.url]?.deadFails ?? 0) >= DEAD_FAIL_THRESHOLD) return false; // 连接级永久失败：兜底也不考虑（否则每次白等一次拒连）
+        return t - err < cooldownMs;
+      })
+      .sort((a, b) => (lastErrorAt[a.url] ?? 0) - (lastErrorAt[b.url] ?? 0));
+  }
+
+  function next() {
+    const avail = available();
+    if (!avail.length) return null;
+    cursor = cursor % avail.length;
+    return avail[cursor++];
+  }
+
+  // Long-lived error memory: a peer keeps its last-error timestamp until a
+  // subsequent success resets it (success clears the failure record) or the
+  // error is no longer in the persist store on next load.
+  async function recordError(url, { status, message } = {}) {
+    if (!url) return;
+    lastErrorAt[url] = now();
+    await persistErrors({ ...lastErrorAt });
+    // 连续 429 计数（仅成功清零）：第 2 次起进入长冷却——该 peer 大概率真被限流，不再频繁试它
+    if (Number(status) === 429) {
+      const prev = stats[url] || {};
+      stats[url] = { ...prev, streak429: (prev.streak429 || 0) + 1 };
+      await persistStats({ ...stats });
+    }
+    // 连接级失败计数（仅成功清零）：第 2 次起进长冷却，且不再被 coolingByLastError 兜底捞回
+    if (isConnLevelFailure(message)) {
+      const prev = stats[url] || {};
+      stats[url] = { ...prev, deadFails: (prev.deadFails || 0) + 1 };
+      await persistStats({ ...stats });
+    }
+  }
+
+  // Outcome of a forwarded request: ok updates the hot-cache (EMA latency,
+  // last successful model) and clears the error memory; failures keep the
+  // error timestamp so the retry pass can order by recovery time.
+  async function recordResult(url, { ok, latencyMs, model } = {}) {
+    if (!url) return;
+    if (ok) {
+      if (lastErrorAt[url] != null) {
+        delete lastErrorAt[url];
+        await persistErrors({ ...lastErrorAt });
+      }
+      const prev = stats[url] || {};
+      stats[url] = {
+        okAt: now(),
+        latencyMs: prev.latencyMs != null && typeof latencyMs === "number"
+          ? Math.round(prev.latencyMs * (1 - EMA_ALPHA) + latencyMs * EMA_ALPHA)
+          : (typeof latencyMs === "number" ? latencyMs : prev.latencyMs ?? 0),
+        fails: 0,
+        model: model || prev.model || "",
+        streak429: 0,
+        deadFails: 0,
+      };
+    } else {
+      const prev = stats[url] || {};
+      stats[url] = { ...prev, fails: (prev.fails || 0) + 1 };
+    }
+    await persistStats({ ...stats });
+  }
+
+  return {
+    all, add, remove, removeByGroup, isCooling, isBroadbandCooling, isBroadbandStale: (url) => {
+      const peer = list.find((p) => p.url === url);
+      return peer ? isBroadbandStale(peer, now(), broadbandStaleMs()) : false;
+    }, isHot, stat, ordered, orderedByLastError, coolingByLastError, available, next,
+    recordError, recordResult, errors: () => ({ ...lastErrorAt }), stats: () => ({ ...stats }),
+  };
+}