mslxdff 0.1.45 → 0.1.55

package/src/routes/chat/index.js

@@ -0,0 +1,178 @@
+import { performance } from "node:perf_hooks";
+import { injectReasoningContent, normalizeModel } from "../../reasoning.js";
+import { isAutoModel } from "../../auto.js";
+import { clientIp, json, readBody, parseHops, summarizePrompt, errMsg } from "../helpers.js";
+import { hedgeDelayMs, shouldHedge } from "../hedge.js";
+import { runHook } from "../../plugins.js";
+import { handleHedge } from "./hedge-handler.js";
+import { handleLocalRelay } from "./local-handler.js";
+import { handlePeerRelay } from "./peer-handler.js";
+import { handleBroadbandRelay } from "./broadband-handler.js";
+import { handleExhaustedLocal, handleExhaustedAll } from "./exhausted-handler.js";
+
+export async function chatHandler({ req, res, upstream, auto, logs, peers, maxHops, groups, bus, token, plugins }) {
+  let body;
+  try {
+    body = await readBody(req);
+  } catch {
+    return json(res, 400, { error: "Invalid JSON body" });
+  }
+
+  if (plugins?.length) {
+    const rc = await runHook(plugins, "request:received", { ip: clientIp(req), hops: parseHops(req.headers["x-mslxdff-hops"]), headers: { "content-type": req.headers["content-type"] }, body });
+    for (const e of rc.errors) logs?.appendEvent?.({ ts: Date.now(), type: "plugin-hook-error", hook: "request:received", plugin: e.plugin, error: e.error });
+    const respond = rc.value?.respond;
+    if (respond && typeof respond === "object") {
+      return json(res, respond.status || 200, respond.body ?? {});
+    }
+  }
+
+  const startedAt = Date.now();
+  const reqId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
+  const perf0 = performance.now();
+  const stages = [];
+  const mark = (name) => stages.push([name, Math.round(performance.now() - perf0)]);
+  const hops = parseHops(req.headers["x-mslxdff-hops"]);
+  const lockModel = req.headers["x-mslxdff-model-lock"] || "";
+  const rawModel = body.model || "";
+  const requested = normalizeModel(lockModel || rawModel || "");
+  const useAuto = isAutoModel(requested);
+  mark("parsed");
+
+  let order;
+  if (lockModel) {
+    order = [requested];
+  } else if (useAuto) {
+    order = auto ? await auto.candidates() : [""];
+  } else {
+    order = auto ? await auto.candidatesFor(requested) : [requested];
+  }
+  if (!order.length) order = [""];
+  const canFallback = order.length > 1;
+  const canForwardPeers = Boolean(peers) && hops < maxHops;
+  mark("ordered");
+
+  const logCall = (model, status) =>
+    logs?.appendCall({ reqId, model, auto: useAuto, status, durationMs: Date.now() - startedAt, stream: Boolean(body.stream), stages });
+  const logError = (model, status, message) =>
+    logs?.appendError({ reqId, model, auto: useAuto, status, message, stages });
+  const evt = (type, data) => {
+    const entry = { ts: Date.now(), reqId, type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
+    if (bus) bus.emit(entry);
+    logs?.appendEvent?.(entry);
+  };
+  const done = (info) => {
+    if (!plugins?.length) return;
+    runHook(plugins, "request:completed", { reqId, requested, useAuto, hops, stream: Boolean(body.stream), durationMs: Date.now() - startedAt, ...info }).catch(() => {});
+  };
+  evt("request", { reqId, hops, ip: clientIp(req), stream: Boolean(body.stream), prompt: summarizePrompt(body), rawModel, requested, lockModel: lockModel || null });
+  evt("ordered", { reqId, order, canFallback, canForwardPeers, useAuto, statuses: auto?.statuses?.() ?? null });
+
+  if (plugins?.length && !lockModel) {
+    const sel = await runHook(plugins, "model:select", { reqId, requested, useAuto, order: [...order], hops, stream: Boolean(body.stream) });
+    if (sel.changed && Array.isArray(sel.value) && sel.value.length) {
+      order = sel.value.filter(Boolean);
+      if (!order.length) order = [requested];
+      evt("plugin-hook", { reqId, hook: "model:select", applied: true, order: [...order] });
+    }
+    for (const e of sel.errors) evt("plugin-hook-error", { reqId, hook: "model:select", plugin: e.plugin, error: e.error });
+  }
+
+  const handlerCtx = { reqId, model: null, body, hops, peers, plugins, evt, logError, logCall };
+
+  let lastErr = null;
+  for (let idx = 0; idx < order.length; idx++) {
+    const model = order[idx];
+    handlerCtx.model = model;
+    evt("model-try", { reqId, model, idx, remaining: order.length - idx });
+    if (plugins?.length) {
+      const bt = await runHook(plugins, "model:beforeTry", { reqId, requested, model, idx, hops });
+      for (const e of bt.errors) evt("plugin-hook-error", { reqId, hook: "model:beforeTry", plugin: e.plugin, error: e.error });
+      if (bt.value === false || bt.value?.skip === true) {
+        evt("plugin-hook", { reqId, hook: "model:beforeTry", applied: true, skipped: model });
+        continue;
+      }
+    }
+    let upRes = null;
+    let forwarded = { ...injectReasoningContent(model, body), model };
+    if (plugins?.length) {
+      const ur = await runHook(plugins, "upstream:request", { reqId, requested, model, payload: forwarded, stream: Boolean(body.stream) });
+      for (const e of ur.errors) evt("plugin-hook-error", { reqId, hook: "upstream:request", plugin: e.plugin, error: e.error });
+      if (ur.changed && ur.value?.payload && typeof ur.value.payload === "object") {
+        forwarded = ur.value.payload;
+        evt("plugin-hook", { reqId, hook: "upstream:request", applied: true, model, rewrittenModel: forwarded.model ?? null });
+      }
+    }
+    const tUp = performance.now();
+    evt("upstream-try", { reqId, model, attempt: idx + 1 });
+    try {
+      upRes = await upstream.chat(forwarded);
+      evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
+    } catch (err) {
+      if (auto) await auto.recordError(model, { message: errMsg(err) });
+      lastErr = { model, upstream: null, status: 502, message: errMsg(err) };
+      logError(model, 502, errMsg(err));
+      evt("upstream-error", { reqId, model, status: 502, message: errMsg(err), timing: err._t ?? { attempts: [], waitMs: 0, totalMs: Math.round(performance.now() - tUp) } });
+    }
+    if (plugins?.length) {
+      runHook(plugins, "upstream:response", {
+        reqId, requested, model,
+        status: upRes instanceof Error ? null : upRes instanceof Object ? (upRes.status ?? null) : null,
+        ok: !(upRes instanceof Error) && upRes ? upRes.status < 400 : false,
+        error: upRes instanceof Error ? errMsg(upRes) : null,
+        timing: upRes?._t ?? null,
+      }).catch(() => {});
+    }
+    mark(`up-${model}`);
+    if (upRes && upRes.status >= 400) {
+      if (auto) await auto.recordError(model, { status: upRes.status });
+      lastErr = { model, upstream: upRes, status: upRes.status, message: null };
+      logError(model, upRes.status, `upstream ${upRes.status}`);
+      evt("upstream-error", { reqId, model, status: upRes.status, message: null, timing: upRes._t ?? null });
+      upRes = null;
+    }
+    if (upRes) {
+      const isStream = Boolean(body.stream);
+      const d = hedgeDelayMs();
+      const hasPeers = Boolean(peers) && peers.ordered().length > 0;
+      const doHedge = shouldHedge({ isStream, canForwardPeers, hedgeDelayMs: d, hasPeers }) && upRes.status === 200 && upRes.body;
+      if (doHedge) {
+        const hr = await handleHedge({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res, hedgeDelayMs: d });
+        if (hr.handled) return;
+        if (hr.lastErr) lastErr = hr.lastErr;
+        if (hr.upRes === null) upRes = null;
+        else if (hr.upRes) upRes = hr.upRes;
+        if (upRes === null) {
+          // hedge 触发后走下面的 peer/broadband 分支，不再走本地串行
+        } else {
+          // hedge 未决出胜负，回退到串行本地
+        }
+      }
+      if (upRes) {
+        const lr = await handleLocalRelay({ upRes, model, body, order, idx, lastErr, requested, useAuto, lockModel, auto, handlerCtx, evt, logCall, logError, mark, perf0, stages, startedAt, plugins, res });
+        if (lr.handled) return;
+        if (lr.lastErr) { lastErr = lr.lastErr; continue; }
+        return;
+      }
+    }
+
+    if (canForwardPeers) {
+      const pr = await handlePeerRelay({ model, body, lastErr, requested, useAuto, lockModel, auto, peers, handlerCtx, evt, logCall, mark, perf0, stages, startedAt, plugins, res });
+      if (pr.handled) return;
+    }
+
+    if (groups) {
+      const br = await handleBroadbandRelay({ model, body, hops, lastErr, requested, useAuto, lockModel, auto, groups, token, bus, logs, handlerCtx, evt, mark, perf0, stages, res, startedAt, plugins });
+      if (br.handled) return;
+    }
+
+    if (canFallback) {
+      evt("fallback", { reqId, from: model, to: order[idx + 1] ?? null, reason: lastErr?.message || `upstream ${lastErr?.status ?? 502}` });
+      continue;
+    }
+    await handleExhaustedLocal({ res, body, lastErr, order, handlerCtx: { ...handlerCtx, model, reqId }, evt, logCall, mark, perf0, stages, done, requested, useAuto });
+    return;
+  }
+
+  await handleExhaustedAll({ res, body, lastErr, order, requested, handlerCtx: { ...handlerCtx, reqId, startedAt }, evt, logCall, mark, perf0, stages });
+}

package/src/routes/chat/local-handler.js

@@ -0,0 +1,88 @@
+import { buildFallbackInfo } from "../fallback.js";
+import { relay, SLOW_TOTAL_MS, STREAM_TIMEOUT_MS, STALL_TIMEOUT_MS, SCORE_STALL_MS } from "../stream.js";
+import { runHook } from "../../plugins.js";
+import { performance } from "node:perf_hooks";
+
+export async function handleLocalRelay({
+  upRes,
+  model,
+  body,
+  order,
+  idx,
+  lastErr,
+  requested,
+  useAuto,
+  lockModel,
+  auto,
+  handlerCtx,
+  evt,
+  logCall,
+  logError,
+  mark,
+  perf0,
+  stages,
+  startedAt,
+  plugins,
+  res,
+}) {
+  logCall(model, upRes.status);
+  const fallback = buildFallbackInfo({ requested, actual: model, lastErr, via: "local", useAuto, lockModel });
+  if (fallback?.fallback) evt("fallback-notice", { reqId: handlerCtx.reqId, requested, actual: model, reason: fallback.reason, notice: fallback.notice, via: "local" });
+  evt("relay-start", { reqId: handlerCtx.reqId, model, via: "local", isStream: Boolean(body.stream), fallback });
+  const out = await relay(res, upRes, body, {
+    fallback,
+    onFirstChunk: (delta) => {
+      mark(`ttf-${model}`);
+      evt("relay-first-chunk", { reqId: handlerCtx.reqId, model, ttfMs: delta });
+      if (plugins?.length) runHook(plugins, "relay:first-chunk", { reqId: handlerCtx.reqId, requested, model, via: "local", ttfMs: delta }).catch(() => {});
+    },
+    onDownstreamAbort: () => {
+      evt("client-abort", { reqId: handlerCtx.reqId, model, totalMs: Math.round(performance.now() - perf0), stages: [...stages] });
+    },
+  });
+  evt("relay-done", { reqId: handlerCtx.reqId, model, via: "local", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+  if (out.status === STREAM_TIMEOUT_MS) {
+    if (auto) await auto.recordError(model, { status: 502, slow: true, note: `stream timeout ${STREAM_TIMEOUT_MS}ms` });
+    const err = { model, upstream: null, status: 502, message: `stream timed out after ${STREAM_TIMEOUT_MS}ms` };
+    logError(model, 502, `stream timeout ${STREAM_TIMEOUT_MS}ms`);
+    evt("upstream-error", { reqId: handlerCtx.reqId, model, status: 502, message: "stream timeout", timing: null });
+    evt("fallback", { reqId: handlerCtx.reqId, from: model, to: order[idx + 1] ?? null, reason: "stream timeout" });
+    return { handled: false, upRes: null, lastErr: err };
+  }
+  if (out.interrupted) {
+    if (auto) {
+      await auto.recordError(model, { status: 200, slow: true, note: `stall ${STALL_TIMEOUT_MS}ms` });
+      await auto.recordLatency(model, out.totalMs ?? (Date.now() - startedAt));
+    }
+    evt("slow-model", { model, elapsedMs: out.totalMs ?? (Date.now() - startedAt), threshold: STALL_TIMEOUT_MS, interrupted: true, detail: out.detail ?? null });
+    logCall(model, 200);
+    evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, interrupted: true, detail: out.detail ?? null, fallback, requested, actual: model });
+    evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, interrupted: true, reqId: handlerCtx.reqId });
+    if (plugins?.length) runHook(plugins, "request:completed", { reqId: handlerCtx.reqId, requested, useAuto, hops: handlerCtx.hops, stream: Boolean(body.stream), durationMs: Date.now() - startedAt, via: "local", status: out.status, actual: model, interrupted: true, fallback }).catch(() => {});
+    return { handled: true };
+  }
+  const elapsed = Date.now() - startedAt;
+  const latencyMs = out.totalMs ?? elapsed;
+  let scoredSlow = false;
+  if (SLOW_TOTAL_MS && auto && elapsed > SLOW_TOTAL_MS && out.status === 200) {
+    void auto.recordError(model, { status: 200, slow: true, note: `slow ${elapsed}ms` });
+    void auto.recordLatency(model, latencyMs);
+    evt("slow-model", { model, elapsedMs: elapsed, threshold: SLOW_TOTAL_MS, reason: "total", detail: out.detail ?? null });
+    scoredSlow = true;
+  }
+  if (out.detail?.stallHits > 0 && auto && out.status === 200) {
+    void auto.recordError(model, { status: 200, slow: true, note: `stall ${out.detail.stallHits}x gap>${SCORE_STALL_MS}ms maxGap ${out.detail.maxGapMs}ms` });
+    void auto.recordLatency(model, latencyMs);
+    evt("slow-model", { model, elapsedMs: elapsed, threshold: SCORE_STALL_MS, reason: "stall", stallHits: out.detail.stallHits, maxGapMs: out.detail.maxGapMs, detail: out.detail ?? null });
+    scoredSlow = true;
+  }
+  if (!scoredSlow && auto && out.status === 200) {
+    await auto.recordOk(model, { latencyMs });
+  } else if (!scoredSlow && auto) {
+    await auto.recordLatency(model, latencyMs);
+  }
+  evt("result", { model, status: out.status, via: "local", timing: upRes._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback, requested, actual: model });
+  evt("client-response", { requested, actual: model, via: "local", fallback, status: out.status, reqId: handlerCtx.reqId });
+  if (plugins?.length) runHook(plugins, "request:completed", { reqId: handlerCtx.reqId, requested, useAuto, hops: handlerCtx.hops, stream: Boolean(body.stream), durationMs: Date.now() - startedAt, via: "local", status: out.status, actual: model, fallback }).catch(() => {});
+  return { handled: true };
+}

package/src/routes/chat/peer-handler.js

@@ -0,0 +1,58 @@
+import { buildFallbackInfo } from "../fallback.js";
+import { relay, SLOW_TOTAL_MS } from "../stream.js";
+import { racePeerCandidates } from "../peers.js";
+import { runHook } from "../../plugins.js";
+
+export async function handlePeerRelay({
+  model,
+  body,
+  lastErr,
+  requested,
+  useAuto,
+  lockModel,
+  auto,
+  peers,
+  handlerCtx,
+  evt,
+  logCall,
+  mark,
+  perf0,
+  stages,
+  startedAt,
+  plugins,
+  res,
+}) {
+  evt("peer-race-start", { reqId: handlerCtx.reqId, model, peers: peers.ordered().length });
+  const win =
+    (await racePeerCandidates(peers.ordered(), handlerCtx)) ||
+    (await racePeerCandidates(peers.orderedByLastError(), handlerCtx));
+  if (!win) {
+    evt("peer-race-lose", { reqId: handlerCtx.reqId, model });
+    return { handled: false };
+  }
+  evt("peer-race-win", { reqId: handlerCtx.reqId, model, winPeer: win.peer.url, winTarget: win.target, latencyMs: win.latencyMs });
+  await peers.recordResult(win.peer.url, { ok: true, latencyMs: win.latencyMs, model: win.target });
+  logCall(win.target, win.res.status);
+  const peerFallback = buildFallbackInfo({ requested, actual: win.target, lastErr, via: "peer", useAuto, lockModel });
+  if (peerFallback?.fallback) evt("fallback-notice", { reqId: handlerCtx.reqId, requested, actual: win.target, reason: peerFallback.reason, notice: peerFallback.notice, via: "peer" });
+  evt("relay-start", { reqId: handlerCtx.reqId, model: win.target, via: "peer", isStream: Boolean(body.stream), fallback: peerFallback });
+  const out = await relay(res, win.res, body, {
+    fallback: peerFallback,
+    onFirstChunk: (d) => mark(`ttf-peer-${win.target}`),
+    onDownstreamAbort: () => evt("client-abort", { reqId: handlerCtx.reqId, model: win.target, totalMs: Math.round(Date.now() - startedAt), stages: [...stages] }),
+  });
+  evt("relay-done", { reqId: handlerCtx.reqId, model: win.target, via: "peer", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
+  if (auto && out.status === 200) {
+    const latencyMs = out.totalMs ?? win.latencyMs;
+    if (out.detail?.stallHits > 0 || (latencyMs && latencyMs > SLOW_TOTAL_MS)) {
+      void auto.recordError(win.target, { status: 200, slow: true, note: `peer slow ${latencyMs}ms` });
+      void auto.recordLatency(win.target, latencyMs);
+    } else {
+      await auto.recordOk(win.target, { latencyMs });
+    }
+  }
+  evt("result", { model: win.target, status: out.status, via: "peer", timing: win.res._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null, fallback: peerFallback, requested, actual: win.target });
+  evt("client-response", { requested, actual: win.target, via: "peer", fallback: peerFallback, status: out.status, reqId: handlerCtx.reqId });
+  if (plugins?.length) runHook(plugins, "request:completed", { reqId: handlerCtx.reqId, requested, useAuto, hops: handlerCtx.hops, stream: Boolean(body.stream), durationMs: Date.now() - startedAt, via: "peer", status: out.status, actual: win.target, fallback: peerFallback }).catch(() => {});
+  return { handled: true };
+}