mslxdff 0.1.152 → 0.1.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/chat-pipeline/index.js +22 -0
- package/src/chat-pipeline/serial-trial.js +2 -1
- package/src/cli/commands/system.js +9 -1
- package/src/logs.js +22 -0
- package/src/model-trace.js +97 -0
- package/src/routes/chat/exhausted-handler.js +4 -0
- package/src/routes/chat/relay-pipeline.js +4 -2
- package/src/routes/peers.js +2 -1
- package/src/routes/stream.js +52 -7
- package/src/runtime/bootstrap.js +3 -2
- package/src/timeline.js +22 -0
- package/src/upstream-engine/sdk/attempt.js +43 -6
package/package.json
CHANGED
|
@@ -4,6 +4,8 @@ import { createEngine } from "./engine.js";
|
|
|
4
4
|
import { runHook } from "../plugins.js";
|
|
5
5
|
import { isFreeModel } from "../models.js";
|
|
6
6
|
import { clientIp, summarizePrompt } from "../routes/helpers.js";
|
|
7
|
+
import { formatTimeline } from "../timeline.js";
|
|
8
|
+
import { summarizeRequest, shouldTraceModel } from "../model-trace.js";
|
|
7
9
|
|
|
8
10
|
/**
|
|
9
11
|
* ChatPipeline 深模块门面 — 对外 execute(req) 单一 inlet
|
|
@@ -25,6 +27,7 @@ export function createChatPipeline({ upstream, auto, logs, peers, groups, bus, t
|
|
|
25
27
|
mark("parsed");
|
|
26
28
|
if (aliasInfo) { try { res?.setHeader?.("x-mslxdff-alias", aliasInfo); } catch {} }
|
|
27
29
|
// mslxdff/ 前缀或 alias 命中时,把 body.model 改写为还原后的模型(与原 gateway 语义一致)
|
|
30
|
+
const timeline = { direct: [], peers: [], retries: 0, result: null };
|
|
28
31
|
if (aliasInfo && req?.body && req.body.model !== requested) {
|
|
29
32
|
req.body = { ...req.body, model: requested };
|
|
30
33
|
}
|
|
@@ -35,6 +38,25 @@ export function createChatPipeline({ upstream, auto, logs, peers, groups, bus, t
|
|
|
35
38
|
const logError = (model, status, message) => logs?.appendError({ reqId, model, auto: useAuto, status, message, stages });
|
|
36
39
|
const evt = (type, data) => {
|
|
37
40
|
const entry = { ts: Date.now(), reqId, type, ...data, model: data.model ?? requested, auto: useAuto, durationMs: Date.now() - startedAt, stages: [...stages] };
|
|
41
|
+
const traceModel = entry.model || requested;
|
|
42
|
+
if (shouldTraceModel(type)) {
|
|
43
|
+
// Note: 模型日志只投影安全字段,不能把含 prompt 的 entry 原样下传 — 见 .agents/notes/implemented/feature/2026-09-25-model-trace-log.md
|
|
44
|
+
const safeTraceData = { ...entry };
|
|
45
|
+
if (safeTraceData.prompt !== undefined) delete safeTraceData.prompt;
|
|
46
|
+
logs?.appendModelTrace?.(traceModel, { type, reqId, model: traceModel, data: safeTraceData, request: type === "request" ? summarizeRequest(req?.body) : null, totalMs: entry.durationMs });
|
|
47
|
+
}
|
|
48
|
+
if (type === "peer-forward") timeline.peers.push({ peer: entry.peer, ok: entry.ok === true, status: entry.status, latencyMs: entry.latencyMs, message: entry.message || entry.error || "" });
|
|
49
|
+
if (type === "upstream-done" || type === "upstream-error") timeline.direct.push({ status: entry.status, reason: entry.message || entry.error || "" });
|
|
50
|
+
if (type === "peer-error") {
|
|
51
|
+
const known = timeline.peers.find((p) => p.peer === entry.peer);
|
|
52
|
+
if (known) { known.ok = false; known.status = entry.status ?? known.status; known.message = entry.message || entry.error || known.message; }
|
|
53
|
+
else timeline.peers.push({ peer: entry.peer, ok: false, status: entry.status, message: entry.message || entry.error || "" });
|
|
54
|
+
}
|
|
55
|
+
if (type === "empty-turn-retry") timeline.retries += 1;
|
|
56
|
+
if (type === "result" || (type === "client-response" && !timeline.result)) {
|
|
57
|
+
timeline.result = { status: entry.status, detail: entry.detail || null };
|
|
58
|
+
try { logs?.appendTimeline?.(formatTimeline({ reqId, model: entry.model || requested, ...timeline, totalMs: Date.now() - startedAt })); } catch {}
|
|
59
|
+
}
|
|
38
60
|
if (bus) bus.emit(entry);
|
|
39
61
|
logs?.appendEvent?.(entry);
|
|
40
62
|
};
|
|
@@ -10,6 +10,7 @@ import { handleBroadbandRelay } from "../routes/chat/broadband-handler.js";
|
|
|
10
10
|
import { handleViaRoute } from "../routes/chat/via-route-handler.js";
|
|
11
11
|
import { handleExhaustedLocal, handleExhaustedAll } from "../routes/chat/exhausted-handler.js";
|
|
12
12
|
import { shouldUseGroupForModel, isHardLocalOnly, isKeyProviderDirectOnly } from "../state/schemas/use-group.js";
|
|
13
|
+
import { summarizeRequest } from "../model-trace.js";
|
|
13
14
|
import { isEmptyTurnError } from "../routes/chat/relay-pipeline.js";
|
|
14
15
|
|
|
15
16
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
@@ -97,7 +98,7 @@ export async function runSerialTrial(ctx, deps = {}) {
|
|
|
97
98
|
let emptyRetried = 0;
|
|
98
99
|
for (;;) {
|
|
99
100
|
const tUp = performance.now();
|
|
100
|
-
evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried });
|
|
101
|
+
evt("upstream-try", { reqId, model, attempt: idx + 1, emptyRetry: emptyRetried, payload: summarizeRequest(forwarded) });
|
|
101
102
|
try {
|
|
102
103
|
upRes = await upstream.chat(forwarded, chatOptsArg);
|
|
103
104
|
evt("upstream-done", { reqId, model, ok: !(upRes instanceof Error) && upRes.status < 400, status: upRes instanceof Error ? null : upRes.status, timing: upRes._t ?? null, error: null });
|
|
@@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url";
|
|
|
4
4
|
import { defaultStateFile } from "../../state.js";
|
|
5
5
|
import { loadToken, refreshToken } from "../../state.js";
|
|
6
6
|
import { stopDaemon, pidFile, logFile } from "../../daemon.js";
|
|
7
|
-
import { logDir, eventsFile, callsFile, errorsFile, recentEvents } from "../../logs.js";
|
|
7
|
+
import { logDir, eventsFile, callsFile, errorsFile, recentEvents, timelineFile, recentTimeline } from "../../logs.js";
|
|
8
8
|
import { fmtEvent } from "../format.js";
|
|
9
9
|
import { fmtShanghaiYMDHMS, fmtShanghaiHMS } from "../../time.js";
|
|
10
10
|
import { printHelp } from "../help.js";
|
|
@@ -95,6 +95,7 @@ export async function handleUninstall(args) {
|
|
|
95
95
|
join(dir, "calls.log"),
|
|
96
96
|
join(dir, "errors.log"),
|
|
97
97
|
join(dir, "events.log"),
|
|
98
|
+
join(dir, "timeline.log"),
|
|
98
99
|
]) {
|
|
99
100
|
try {
|
|
100
101
|
rmSync(f, { force: true });
|
|
@@ -128,6 +129,13 @@ export async function handleLog(args) {
|
|
|
128
129
|
if (count <= 10) {
|
|
129
130
|
console.log(`\nhint: mslxdff -log 100 | calls: ${callsFile()} errors: ${errorsFile()} daemon: ${logFile()}`);
|
|
130
131
|
}
|
|
132
|
+
const timeline = recentTimeline(count);
|
|
133
|
+
if (timeline.length) {
|
|
134
|
+
console.log(`--- last ${timeline.length} timeline line(s) ---`);
|
|
135
|
+
for (const line of timeline) console.log(line);
|
|
136
|
+
}
|
|
137
|
+
console.log(`timeline: ${timelineFile()}`);
|
|
138
|
+
console.log(`model logs: ${dir}\\<provider>-<model>.log`);
|
|
131
139
|
process.exit(0);
|
|
132
140
|
}
|
|
133
141
|
|
package/src/logs.js
CHANGED
|
@@ -25,6 +25,10 @@ export function eventsFile() {
|
|
|
25
25
|
return join(logDir(), "events.log");
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
export function timelineFile() {
|
|
29
|
+
return join(logDir(), "timeline.log");
|
|
30
|
+
}
|
|
31
|
+
|
|
28
32
|
function ensureDir(dir) {
|
|
29
33
|
mkdirSync(dir, { recursive: true });
|
|
30
34
|
}
|
|
@@ -86,6 +90,13 @@ function appendLine(file, entry) {
|
|
|
86
90
|
.then(() => trimIfOversizedAsync(file).catch(() => {}));
|
|
87
91
|
}
|
|
88
92
|
|
|
93
|
+
export function recentTimeline(n = 10, { file = timelineFile() } = {}) {
|
|
94
|
+
try {
|
|
95
|
+
if (!existsSync(file)) return [];
|
|
96
|
+
return readFileSync(file, "utf8").split("\n").filter(Boolean).slice(-n);
|
|
97
|
+
} catch { return []; }
|
|
98
|
+
}
|
|
99
|
+
|
|
89
100
|
export function appendCall(entry, { file = callsFile() } = {}) {
|
|
90
101
|
appendLine(file, entry);
|
|
91
102
|
}
|
|
@@ -132,3 +143,14 @@ export function lastError({ file = errorsFile() } = {}) {
|
|
|
132
143
|
export function recentErrors(n = 5, { file = errorsFile() } = {}) {
|
|
133
144
|
return readLines(file).slice(-n);
|
|
134
145
|
}
|
|
146
|
+
|
|
147
|
+
export function appendTimeline(entry, { file = timelineFile() } = {}) {
|
|
148
|
+
const line = `${fmtShanghaiYMDHMS(new Date())} ${String(entry || "")}\n`;
|
|
149
|
+
ensureDir(dirname(file));
|
|
150
|
+
if (shouldSync(file)) {
|
|
151
|
+
appendFileSync(file, line);
|
|
152
|
+
trimIfOversized(file);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
appendFile(file, line).catch(() => {}).then(() => trimIfOversizedAsync(file).catch(() => {}));
|
|
156
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Per-model request trace formatting. Never include prompt, response body, headers or credentials.
|
|
2
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { logDir } from "./logs.js";
|
|
5
|
+
import { fmtShanghaiYMDHMS } from "./time.js";
|
|
6
|
+
|
|
7
|
+
const MAX_TRACE_BYTES = 1024 * 1024;
|
|
8
|
+
|
|
9
|
+
export function modelLogName(model) {
|
|
10
|
+
const s = String(model || "unknown").trim().toLowerCase();
|
|
11
|
+
const safe = s.replace(/[^a-z0-9._-]+/g, "-").replace(/^[-.]+|[-.]+$/g, "").slice(0, 180);
|
|
12
|
+
return `${safe || "unknown"}.log`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function modelLogFile(model) {
|
|
16
|
+
return join(logDir(), modelLogName(model));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
function count(v) {
|
|
21
|
+
const n = Number(v);
|
|
22
|
+
return Number.isFinite(n) && n >= 0 ? n : null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function summarizeRequest(body = {}) {
|
|
26
|
+
const messages = Array.isArray(body.messages) ? body.messages : [];
|
|
27
|
+
const roles = {};
|
|
28
|
+
for (const m of messages) roles[String(m?.role || "unknown")] = (roles[String(m?.role || "unknown")] || 0) + 1;
|
|
29
|
+
return {
|
|
30
|
+
stream: body.stream === true,
|
|
31
|
+
messages: messages.length,
|
|
32
|
+
roles,
|
|
33
|
+
tools: Array.isArray(body.tools) ? body.tools.length : 0,
|
|
34
|
+
maxTokens: count(body.max_tokens ?? body.max_completion_tokens),
|
|
35
|
+
temperature: Number.isFinite(Number(body.temperature)) ? Number(body.temperature) : null,
|
|
36
|
+
topP: Number.isFinite(Number(body.top_p)) ? Number(body.top_p) : null,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function hostPort(value) {
|
|
41
|
+
try { const u = new URL(String(value)); return u.port ? `${u.hostname}:${u.port}` : u.hostname; } catch { return "-"; }
|
|
42
|
+
}
|
|
43
|
+
const TRACE_STAGES = new Set([
|
|
44
|
+
"request", "alias", "ordered", "model-try", "upstream-try", "upstream-done", "upstream-error",
|
|
45
|
+
"peer-request", "peer-forward", "peer-error", "relay-start", "relay-first-chunk", "relay-done",
|
|
46
|
+
"client-response", "result", "exhausted-local", "exhausted-all",
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
export function shouldTraceModel(type) {
|
|
50
|
+
return TRACE_STAGES.has(String(type || ""));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function safeText(value, n = 140) {
|
|
54
|
+
return String(value ?? "")
|
|
55
|
+
.replace(/([?&](?:token|refreshToken|access[_-]?token|refresh[_-]?token|api[_-]?key|apikey|key|secret|password|cookie)=)[^&\s]+/gi, "$1[redacted]")
|
|
56
|
+
.replace(/(authorization|bearer|access[_-]?token|refresh[_-]?token|cookie|api[-_]?key|password|secret)\s*[:=]\s*[^,; ]+/gi, "$1=[redacted]")
|
|
57
|
+
.replace(/\s+/g, " ").trim().slice(0, n);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function kv(obj) {
|
|
61
|
+
return Object.entries(obj).filter(([, v]) => v != null && v !== "").map(([k, v]) => `${k}=${typeof v === "object" ? JSON.stringify(v) : v}`).join(" ");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function formatModelTrace({ type, reqId, model, data = {}, request = null, totalMs = null } = {}) {
|
|
65
|
+
const base = { time: fmtShanghaiYMDHMS(new Date()), req: reqId || "-", model: model || "-", stage: type || "-" };
|
|
66
|
+
let detail = "";
|
|
67
|
+
if (type === "request") detail = `incoming ${kv({ ...(request || summarizeRequest(data.body)), hops: data.hops, useAuto: data.useAuto })}`;
|
|
68
|
+
else if (type === "ordered") detail = `route order=${(data.order || []).join(" | ")} hops=${data.hops ?? "-"} fallback=${data.canFallback ? 1 : 0}`;
|
|
69
|
+
else if (type === "model-try") detail = `target=${data.model || model || "-"} idx=${data.idx ?? "-"} remaining=${data.remaining ?? "-"}`;
|
|
70
|
+
else if (type === "alias") detail = `rawModel=${data.rawModel || "-"} requested=${data.requested || model || "-"}`;
|
|
71
|
+
else if (type === "exhausted-local" || type === "exhausted-all") detail = `last=${data.lastModel || model || "-"} status=${data.lastStatus ?? "-"} order=${(data.order || []).join(" | ")}`;
|
|
72
|
+
else if (type === "upstream-try") detail = `target=${data.model || model || "-"} attempt=${data.attempt ?? "-"} payload=${kv(data.payload || {})}`;
|
|
73
|
+
else if (type === "upstream-done" || type === "upstream-error") detail = `upstream status=${data.status ?? "-"} timing=${data.timing?.totalMs ?? "-"}ms ${safeText(data.message || data.error || "")}`;
|
|
74
|
+
else if (type === "peer-request" || type === "peer-forward" || type === "peer-error") detail = `peer=${hostPort(data.peer)} status=${data.status ?? "-"} latency=${data.latencyMs ?? "-"}ms payload=${kv(data.payload || {})} ${safeText(data.message || data.error || "")}`;
|
|
75
|
+
else if (type === "relay-done") {
|
|
76
|
+
const d = data.detail || {};
|
|
77
|
+
const u = d.usage || {};
|
|
78
|
+
detail = `upstream_response status=${data.status ?? "-"} finish=${d.sawFinishReason || "-"} chunks=${d.receivedChunks ?? "-"} bytes=${d.receivedBytes ?? "-"} tools=${d.toolCalls ?? 0} chars=${d.chars ?? 0} usage(prompt=${u.prompt_tokens ?? "-"}/completion=${u.completion_tokens ?? "-"}) elapsed=${data.totalMs ?? "-"}ms`;
|
|
79
|
+
} else if (type === "result" || type === "client-response") {
|
|
80
|
+
detail = `client status=${data.status ?? "-"} via=${data.via || "-"} actual=${data.actual || model || "-"} fallback=${data.fallback?.fallback ? 1 : 0} total=${totalMs ?? data.durationMs ?? "-"}ms`;
|
|
81
|
+
} else detail = safeText(data.message || data.reason || data.error || "");
|
|
82
|
+
return `[${base.time}] [req=${base.req}] [model=${base.model}] [stage=${base.stage}] ${detail}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function appendModelTrace(model, entry, { file = modelLogFile(model) } = {}) {
|
|
86
|
+
const line = `${formatModelTrace(entry)}\n`;
|
|
87
|
+
try {
|
|
88
|
+
mkdirSync(logDir(), { recursive: true });
|
|
89
|
+
if (existsSync(file) && statSync(file).size > MAX_TRACE_BYTES) {
|
|
90
|
+
const lines = readFileSync(file, "utf8").split("\n").filter(Boolean);
|
|
91
|
+
writeFileSync(file, lines.slice(-100).join("\n") + "\n");
|
|
92
|
+
}
|
|
93
|
+
appendFileSync(file, line);
|
|
94
|
+
} catch {
|
|
95
|
+
// Logging must never affect the chat request.
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -15,6 +15,7 @@ export async function handleExhaustedLocal({ res, body, lastErr, order, handlerC
|
|
|
15
15
|
});
|
|
16
16
|
evt("relay-done", { reqId: handlerCtx.reqId, model: lastErr.model, via: "local-exhausted", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
|
|
17
17
|
evt("result", { reqId: handlerCtx.reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
|
|
18
|
+
evt("client-response", { requested, actual: lastErr.model, via: "local", fallback: false, status: out.status, reqId: handlerCtx.reqId });
|
|
18
19
|
// 最后一站:relay 超时路径不写响应(设计留给上层 failover),这里没有上层,必须自己收尾
|
|
19
20
|
if (out.timedOut) {
|
|
20
21
|
json(res, 502, { error: out.detail?.upstreamError ? `upstream error: ${out.detail.upstreamError}` : `stream timed out after ${out.totalMs}ms` });
|
|
@@ -22,6 +23,7 @@ export async function handleExhaustedLocal({ res, body, lastErr, order, handlerC
|
|
|
22
23
|
return true;
|
|
23
24
|
}
|
|
24
25
|
evt("result", { reqId: handlerCtx.reqId, model, status: lastErr?.status ?? 502, via: "none", timing: null });
|
|
26
|
+
evt("client-response", { requested, actual: model, via: "none", fallback: false, status: lastErr?.status ?? 502, reqId: handlerCtx.reqId });
|
|
25
27
|
done({ via: "none", status: lastErr?.status ?? 502, error: lastErr?.message || "all auto models failed" });
|
|
26
28
|
json(res, 502, { error: lastErr?.message || "all auto models failed" });
|
|
27
29
|
return true;
|
|
@@ -39,6 +41,7 @@ export async function handleExhaustedAll({ res, body, lastErr, order, requested,
|
|
|
39
41
|
});
|
|
40
42
|
evt("relay-done", { reqId: handlerCtx.reqId, model: lastErr.model, via: "local-final", status: out.status, ttfMs: out.ttfMs, totalMs: out.totalMs, aborted: out.aborted, interrupted: out.interrupted ?? false, detail: out.detail ?? null });
|
|
41
43
|
evt("result", { reqId: handlerCtx.reqId, model: lastErr.model, status: out.status, via: "local", timing: lastErr.upstream._t ?? null, ttfMs: out.ttfMs, totalMs: out.totalMs, detail: out.detail ?? null });
|
|
44
|
+
evt("client-response", { requested, actual: lastErr.model, via: "local", fallback: false, status: out.status, reqId: handlerCtx.reqId });
|
|
42
45
|
// 最后一站:relay 超时路径不写响应,这里没有上层 failover,必须自己收尾
|
|
43
46
|
if (out.timedOut) {
|
|
44
47
|
json(res, 502, { error: out.detail?.upstreamError ? `upstream error: ${out.detail.upstreamError}` : `stream timed out after ${out.totalMs}ms` });
|
|
@@ -46,6 +49,7 @@ export async function handleExhaustedAll({ res, body, lastErr, order, requested,
|
|
|
46
49
|
return true;
|
|
47
50
|
}
|
|
48
51
|
evt("result", { reqId: handlerCtx.reqId, model: lastErr?.model ?? requested, status: lastErr?.status ?? 502, via: "none", timing: null });
|
|
52
|
+
evt("client-response", { requested, actual: lastErr?.model ?? requested, via: "none", fallback: false, status: lastErr?.status ?? 502, reqId: handlerCtx.reqId });
|
|
49
53
|
json(res, 502, { error: lastErr?.message || "all auto models failed" });
|
|
50
54
|
return true;
|
|
51
55
|
}
|
|
@@ -137,10 +137,12 @@ export function createRelayPipeline({
|
|
|
137
137
|
!["tool_calls", "function_call"].includes(_d.sawFinishReason);
|
|
138
138
|
if (_emptyTurn) {
|
|
139
139
|
const _why = _d.sawFinishReason ? ` (finish_reason=${_d.sawFinishReason})` : " (no content, no tool calls)";
|
|
140
|
-
|
|
140
|
+
// 错误包络暂扣后下游不再直观看到上游原文:把摘要带进最终报错(截断 200 字),排障不断线。
|
|
141
|
+
const _err = _d.upstreamErrorText ? ` upstream=${String(_d.upstreamErrorText).slice(0, 200)}` : "";
|
|
142
|
+
try { _logError(actual, 502, `empty turn${_why}${_err}`); } catch {}
|
|
141
143
|
_evt("upstream-error", { reqId, model: actual, status: 502, message: "empty turn", timing: null });
|
|
142
144
|
_evt("fallback", { reqId, from: actual, to: null, reason: "empty turn" });
|
|
143
|
-
return { handled: false, upRes: null, lastErr: { model: actual, upstream: null, status: 502, message: `EMPTY_MODEL_RESPONSE: upstream returned 200 with no content${_why} — retry or rephrase` } };
|
|
145
|
+
return { handled: false, upRes: null, lastErr: { model: actual, upstream: null, status: 502, message: `EMPTY_MODEL_RESPONSE: upstream returned 200 with no content${_why}${_err} — retry or rephrase` } };
|
|
144
146
|
}
|
|
145
147
|
|
|
146
148
|
// 5a. 首块超时未写字节 → 回退(显式 timedOut 字段,status 只是 HTTP 语义展示)
|
package/src/routes/peers.js
CHANGED
|
@@ -12,6 +12,7 @@ const DEFAULT_PEER_CONNECT_TIMEOUT_MS = 3_000;
|
|
|
12
12
|
// 沿用 30s 会把组内转发的中继活流掐成 "terminated"(实测首块后 ~31s 必断,本地直连
|
|
13
13
|
// 无此限制——组内因此比直连"不流畅")。与应用层 MAX_STREAM_MS(120s) 对齐:
|
|
14
14
|
// undici 层只做最后兜底,掐流交给应用层的首块/总时长策略。
|
|
15
|
+
import { summarizeRequest } from "../model-trace.js";
|
|
15
16
|
const PEER_BODY_TIMEOUT_MS = 120_000;
|
|
16
17
|
export { PEER_BODY_TIMEOUT_MS };
|
|
17
18
|
|
|
@@ -203,7 +204,7 @@ export async function racePeerCandidates(candidates, ctx) {
|
|
|
203
204
|
tried.add(peer.url);
|
|
204
205
|
const ctrl = new AbortController();
|
|
205
206
|
ctrls.set(peer.url, ctrl);
|
|
206
|
-
ctx.evt("peer-request", { peer: peer.url, model: target, hops: ctx.hops + 1 });
|
|
207
|
+
ctx.evt("peer-request", { peer: peer.url, model: target, hops: ctx.hops + 1, payload: summarizeRequest(ctx.body) });
|
|
207
208
|
// 插件 hook:peer:beforeForward — 转发给组员前观察
|
|
208
209
|
if (ctx.plugins?.length) {
|
|
209
210
|
runHook(ctx.plugins, "peer:beforeForward", { reqId: ctx.reqId, peer: peer.url, model: target, hops: ctx.hops + 1 }).catch(() => {});
|
package/src/routes/stream.js
CHANGED
|
@@ -30,6 +30,40 @@ function hasPayload(text) {
|
|
|
30
30
|
return false;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
// 错误包络 chunk 判定(纯函数):data 行 JSON 含顶层 .error 对象、或只有 finish_reason=error
|
|
34
|
+
// 的空帧,且整 chunk 无任何 content/tool_calls/reasoning 输出 → 返回错误摘要(建议暂扣),
|
|
35
|
+
// 否则返回 null(透传)。解析失败一律透传(默认保安全)。
|
|
36
|
+
// 背景:网关把上游失败包成 HTTP 200 SSE;暂扣后下游流式 UI 不再展示瞬时错误
|
|
37
|
+
//(如 Vertex 503 + google fallback 400),本轮走空转重试,客户端只看到最终结果。
|
|
38
|
+
function holdableChunk(txt) {
|
|
39
|
+
if (typeof txt !== "string" || !txt.includes("data:")) return null;
|
|
40
|
+
let sawData = false;
|
|
41
|
+
const errs = [];
|
|
42
|
+
for (const line of txt.split("\n")) {
|
|
43
|
+
const t = line.trim();
|
|
44
|
+
if (!t.startsWith("data:")) continue;
|
|
45
|
+
const d = t.slice(5).trim();
|
|
46
|
+
if (!d || d === "[DONE]") continue;
|
|
47
|
+
let j = null;
|
|
48
|
+
try { j = JSON.parse(d); } catch { return null; }
|
|
49
|
+
if (!j || typeof j !== "object") return null;
|
|
50
|
+
sawData = true;
|
|
51
|
+
const c0 = (Array.isArray(j.choices) && j.choices[0]) || {};
|
|
52
|
+
const delta = c0.delta || {};
|
|
53
|
+
const msg = c0.message || {};
|
|
54
|
+
const out = delta.content || msg.content || delta.tool_calls || msg.tool_calls || delta.reasoning || msg.reasoning;
|
|
55
|
+
if (out && !(Array.isArray(out) && out.length === 0)) return null;
|
|
56
|
+
if (j.error && typeof j.error === "object") {
|
|
57
|
+
const m = j.error.message || j.error.code || "";
|
|
58
|
+
if (m) errs.push(String(m).slice(0, 300));
|
|
59
|
+
} else if (c0.finish_reason !== "error") {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (!sawData) return null;
|
|
64
|
+
return errs.length ? errs.join(" | ") : "finish_reason=error 空帧";
|
|
65
|
+
}
|
|
66
|
+
|
|
33
67
|
// 自持 reader 优先:for-await 会锁定 ReadableStream,使 body.cancel() 必 reject(真流上等于空操作),
|
|
34
68
|
// 超时/断下游要真能掐上游必须走 reader.cancel()
|
|
35
69
|
async function* bodyChunks(body, reader) {
|
|
@@ -116,6 +150,8 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
116
150
|
chars: 0,
|
|
117
151
|
toolCalls: 0,
|
|
118
152
|
chatShaped: false,
|
|
153
|
+
heldErrorChunks: 0,
|
|
154
|
+
upstreamErrorText: null,
|
|
119
155
|
recoveries: 0,
|
|
120
156
|
};
|
|
121
157
|
let prevChunkAt = t0;
|
|
@@ -206,6 +242,13 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
206
242
|
if (gap > SCORE_STALL_MS) detail.stallHits += 1;
|
|
207
243
|
prevChunkAt = now;
|
|
208
244
|
const txt = chunkText(chunk);
|
|
245
|
+
// 错误包络暂扣:本轮尚未写出真实输出时错误帧不写下游(下游 UI 不再展示瞬时错误),
|
|
246
|
+
// 摘要记 detail.upstreamErrorText 供空转判定与最终报错;已写出真实内容后的错误帧照常透传。
|
|
247
|
+
const holdErr = !wrotePayload ? holdableChunk(txt) : null;
|
|
248
|
+
if (holdErr) {
|
|
249
|
+
detail.heldErrorChunks += 1;
|
|
250
|
+
if (!detail.upstreamErrorText) detail.upstreamErrorText = String(holdErr).slice(0, 500);
|
|
251
|
+
}
|
|
209
252
|
const isPayload = hasPayload(txt);
|
|
210
253
|
try {
|
|
211
254
|
if (txt.includes("[DONE]")) detail.sawDone = true;
|
|
@@ -252,7 +295,7 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
252
295
|
// 首块/空闲超时后上游仍吐出了真实数据 → 只是慢,不是死:撤销超时判定,照常转发
|
|
253
296
|
//(cancel 是异步的,竞态窗口内已到达的数据是纯收益;丢掉是纯损失)
|
|
254
297
|
// 注释帧(keepalive)不算:否则对端只要在发心跳,闸门就永远解除
|
|
255
|
-
if (isPayload && (timedOut || stalled)) {
|
|
298
|
+
if (isPayload && !holdErr && (timedOut || stalled)) {
|
|
256
299
|
timedOut = false;
|
|
257
300
|
stalled = false;
|
|
258
301
|
detail.recoveries = (detail.recoveries || 0) + 1;
|
|
@@ -260,7 +303,7 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
260
303
|
if (firstTimer) { clearTimeout(firstTimer); firstTimer = null; }
|
|
261
304
|
}
|
|
262
305
|
if (timedOut || stalled || tooLong) break;
|
|
263
|
-
if (isPayload && first) {
|
|
306
|
+
if (isPayload && first && !holdErr) {
|
|
264
307
|
first = false;
|
|
265
308
|
ttf = Math.round(now - t0);
|
|
266
309
|
onFirstChunk?.(ttf);
|
|
@@ -279,11 +322,13 @@ export async function relay(res, upRes, body, { onFirstChunk, onDownstreamAbort,
|
|
|
279
322
|
}
|
|
280
323
|
} catch {}
|
|
281
324
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
325
|
+
if (!holdErr) {
|
|
326
|
+
wroteAny = true;
|
|
327
|
+
if (isPayload) wrotePayload = true;
|
|
328
|
+
detail.wroteChunks += 1;
|
|
329
|
+
detail.wroteBytes += Buffer.isBuffer(outChunk) ? outChunk.length : (outChunk?.length ?? len);
|
|
330
|
+
try { res.write(outChunk); } catch { /* 下游已断开:onClose 已掐上游 */ }
|
|
331
|
+
}
|
|
287
332
|
armStall();
|
|
288
333
|
}
|
|
289
334
|
if (!detail.exitReason) detail.exitReason = detail.downstreamClosed ? "downstream-closed" : "normal";
|
package/src/runtime/bootstrap.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { appendCall, appendError, appendEvent } from "../logs.js";
|
|
1
|
+
import { appendCall, appendError, appendEvent, appendTimeline } from "../logs.js";
|
|
2
2
|
import { setupProviders } from "./providers-setup.js";
|
|
3
3
|
import { startServerLifecycle } from "./server-lifecycle.js";
|
|
4
|
+
import { appendModelTrace } from "../model-trace.js";
|
|
4
5
|
import { startGroupSync } from "./group-sync.js";
|
|
5
6
|
import { startBroadband } from "./broadband.js";
|
|
6
7
|
|
|
7
|
-
const logs = { appendCall, appendError, appendEvent };
|
|
8
|
+
const logs = { appendCall, appendError, appendEvent, appendTimeline, appendModelTrace };
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* daemon 启动门面 — 仅编排:组装世界 → 服务生命周期 → 群组同步 → 宽带中继 → 自更新。
|
package/src/timeline.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// 人读请求时间线:只输出状态/耗时/出口,不输出 prompt、响应正文或凭据。
|
|
2
|
+
function hostPort(value) {
|
|
3
|
+
try { const u = new URL(String(value)); return u.port ? `${u.hostname}:${u.port}` : u.hostname; } catch { return String(value || "-").slice(0, 80); }
|
|
4
|
+
}
|
|
5
|
+
function short(value, n = 90) {
|
|
6
|
+
const s = String(value ?? "").replace(/\s+/g, " ").trim();
|
|
7
|
+
return !s ? "-" : s.length > n ? `${s.slice(0, n - 1)}…` : s;
|
|
8
|
+
}
|
|
9
|
+
function outcome(status, reason) {
|
|
10
|
+
if (status == null) return reason ? `- ${short(reason)}` : "-";
|
|
11
|
+
return `${status}${reason ? ` ${short(reason)}` : ""}`;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function formatTimeline({ reqId, model, direct = [], peers = [], retries = 0, result = null, totalMs = 0 } = {}) {
|
|
15
|
+
const d = direct.length ? direct[direct.length - 1] : null;
|
|
16
|
+
const ps = peers.map((p) => `[peer=${hostPort(p.peer)} ${p.ok ? "win" : "fail"} ${p.latencyMs != null ? `${p.latencyMs}ms` : "timing-"} ${short(p.message || p.status || "", 70)}]`);
|
|
17
|
+
const r = result || {};
|
|
18
|
+
const detail = r.detail || {};
|
|
19
|
+
const res = r.status == null ? "-" : `${r.status}${detail.sawFinishReason ? ` ${detail.sawFinishReason}` : ""}${detail.toolCalls ? ` tools=${detail.toolCalls}` : ""}${detail.chars != null ? ` chars=${detail.chars}` : ""}${detail.interrupted ? " interrupted=1" : ""}${detail.timedOut ? " timedOut=1" : ""}`;
|
|
20
|
+
const directText = d ? outcome(d.status, d.reason) : "-";
|
|
21
|
+
return `[req=${reqId || "-"}] [model=${model || "-"}] [direct=${directText}]${ps.length ? ` ${ps.join(" ")}` : ""}${retries ? ` [retry=${retries}]` : ""} [result=${res}] [total=${Math.max(0, Math.round(Number(totalMs) || 0))}ms]`;
|
|
22
|
+
}
|
|
@@ -4,10 +4,39 @@
|
|
|
4
4
|
// HTTP 错误就地映射为带状态码的 Response;装载失败抛 _sdkLoadFailed 由引擎回退 legacy。
|
|
5
5
|
// responses 适配器(sdk/responses.js)复用本文件的 baseURL 解析、错误映射与流序列化。
|
|
6
6
|
// 见 .scratch/ai-sdk-upstream/{SPEC.md,SPEC-p3-responses.md} 与 docs/adr/0017。
|
|
7
|
+
// Headers 超时(防 SDK 通道挂死):doStream 裸 await 曾因上游连接半死永不 resolve
|
|
8
|
+
//(cline muse-spark 2026-09-25 17:48 悬空 27min+,无 upstream-done/error/result)。
|
|
9
|
+
// 默认 120s(覆盖慢模型首块握手的合理上限),MSLXDFF_SDK_HEADERS_TIMEOUT_MS=0 关闭。
|
|
10
|
+
// 注意错误文案不得含 "timed out":cline runChat 靠该子串做网络重试,命中会把挂死放大 3 倍。
|
|
7
11
|
import { toModelPrompt, toModelTools, toModelToolChoice, toModelParams } from "./convert.js";
|
|
8
12
|
import { createSseSerializer } from "./sse.js";
|
|
9
13
|
import { diagnoseToolSequence, compactSequence } from "./diagnose.js";
|
|
10
14
|
import { getUndici } from "../../compat.js";
|
|
15
|
+
function headersTimeoutMs(env = process.env) {
|
|
16
|
+
const n = Number(env.MSLXDFF_SDK_HEADERS_TIMEOUT_MS);
|
|
17
|
+
return Number.isFinite(n) && n >= 0 ? n : 120_000;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// race 包裹:超时 reject mkErr();先赢路径清定时器并吞掉迟到的 rejection(防 unhandled)。
|
|
21
|
+
export function withHeadersTimeout(promise, ms, mkErr) {
|
|
22
|
+
if (!Number.isFinite(ms) || ms <= 0) return promise;
|
|
23
|
+
let timer = null;
|
|
24
|
+
const guard = new Promise((_, reject) => {
|
|
25
|
+
timer = setTimeout(() => reject(mkErr()), ms);
|
|
26
|
+
});
|
|
27
|
+
return Promise.race([
|
|
28
|
+
Promise.resolve(promise).then(
|
|
29
|
+
(v) => { clearTimeout(timer); return v; },
|
|
30
|
+
(e) => { clearTimeout(timer); throw e; },
|
|
31
|
+
),
|
|
32
|
+
guard,
|
|
33
|
+
]).catch((e) => {
|
|
34
|
+
// 超时赢后原 promise 仍可能 reject:静默兜住,不让其变成 unhandled rejection。
|
|
35
|
+
if (promise && typeof promise.catch === "function") promise.catch(() => {});
|
|
36
|
+
throw e;
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
11
40
|
|
|
12
41
|
let sdkPromise = null;
|
|
13
42
|
|
|
@@ -155,15 +184,23 @@ export async function attemptOnceSdk({
|
|
|
155
184
|
...(capturedFetch ? { fetch: capturedFetch } : {}),
|
|
156
185
|
});
|
|
157
186
|
const model = provider.chatModel(String(body?.model || ""));
|
|
187
|
+
const htMs = headersTimeoutMs();
|
|
188
|
+
const aborter = htMs > 0 ? new AbortController() : null;
|
|
158
189
|
let res;
|
|
159
190
|
try {
|
|
160
|
-
res = await
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
191
|
+
res = await withHeadersTimeout(
|
|
192
|
+
model.doStream({
|
|
193
|
+
prompt: toModelPrompt(body?.messages),
|
|
194
|
+
...toModelParams(body, providerName),
|
|
195
|
+
tools: toModelTools(body?.tools),
|
|
196
|
+
toolChoice: toModelToolChoice(body?.tool_choice),
|
|
197
|
+
...(aborter ? { abortSignal: aborter.signal } : {}),
|
|
198
|
+
}),
|
|
199
|
+
htMs,
|
|
200
|
+
() => new Error(`sdk-channel: headers timeout after ${htMs}ms(上游未返回响应头,防挂死;MSLXDFF_SDK_HEADERS_TIMEOUT_MS=0 关闭)`),
|
|
201
|
+
);
|
|
166
202
|
} catch (e) {
|
|
203
|
+
try { aborter?.abort(); } catch {}
|
|
167
204
|
const mapped = errorResponseFromSdkError(e, { marker });
|
|
168
205
|
if (mapped) {
|
|
169
206
|
try { logRequestDiagnosis(lastBodyText, mapped.status); } catch {}
|