mslxdff 0.1.159 → 0.1.161

package/src/talk-log.js

@@ -0,0 +1,226 @@
+// 环形对话日志（talk log）：按「供应商-模型-talk.log」分文件，只保留最近 1 小时的完整对话。
+// 记三件事：我问了什么 / 模型怎么想（reasoning）/ 模型怎么答（content），供人工调研模型的回答方式。
+// 与 model-trace 相反：那边刻意不落正文（「加日志」≠「倒数据」），这里专门落正文 —— 所以必须脱敏 + 封顶。
+// 默认开；MSLXDFF_TALK_LOG=0 关闭，窗口 MSLXDFF_TALK_LOG_WINDOW_MIN（分，默认 60），单文件 MSLXDFF_TALK_LOG_MAX_KB（默认 5MB）。
+import { appendFileSync, chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readdirSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { logDir } from "./logs.js";
+import { fmtShanghaiYMDHMS } from "./time.js";
+
+const SUBDIR = "talk";
+const DEFAULT_WINDOW_MIN = 60;
+const DEFAULT_MAX_KB = 5 * 1024; // 5MB per talk-log file (ring buffer)
+const MARK = "#talk-entry";
+// 正常路径只 append；整文件重写（环形淘汰）只在「最老条目出窗」或「超字节上限」时发生。
+// oldestCache 记住每个文件当前最老的 ts，避免每次写入都回读全文件。
+const oldestCache = new Map();
+
+export function talkLogEnabled() { return String(process.env.MSLXDFF_TALK_LOG ?? "1") !== "0"; }
+function windowMs() { const n = Number(process.env.MSLXDFF_TALK_LOG_WINDOW_MIN); return Number.isFinite(n) && n > 0 ? n * 60000 : DEFAULT_WINDOW_MIN * 60000; }
+function maxBytes() { const n = Number(process.env.MSLXDFF_TALK_LOG_MAX_KB); return Number.isFinite(n) && n > 0 ? n * 1024 : DEFAULT_MAX_KB * 1024; }
+/** 仅测试用：进程重启等价物（环形窗口的内存基线清空）。 */
+export function resetTalkLogCache() { oldestCache.clear(); }
+
+/** 供应商-模型-talk.log：qoder/qfmodel → qoder-qfmodel-talk.log；裸 id（免费池）直接用一个名字段。 */
+export function talkLogName(model) {
+  const s = String(model || "unknown").trim().toLowerCase();
+  const segs = s.includes("/") ? s.split("/").filter((x) => x && x !== "unknown") : [s];
+  const safe = segs.join("-").replace(/[^a-z0-9._-]+/g, "-").replace(/^[-.]+|[-.]+$/g, "").slice(0, 180);
+  return `${safe || "unknown"}-talk.log`;
+}
+
+export function talkDir() { return join(logDir(), SUBDIR); }
+export function talkLogFile(model) { return join(talkDir(), talkLogName(model)); }
+
+// 正文里的凭据一律先抹再落盘（保留换行，不像 model-trace 那样压成一行 —— 这里要读的是回答本身）。
+const MASKS = [
+  [/([?&](?:token|refresh[_-]?token|access[_-]?token|api[_-]?key|apikey|key|secret|password|cookie)=)[^\s&]+/gi, "$1[已脱敏]"],
+  [/(authorization|bearer|proxy-authorization|cookie|x-api-key)\s*[:=]\s*[^\s,;]+/gi, "$1=[已脱敏]"],
+  [/("(?:apiKey|api_key|accessToken|refreshToken|secret|password|client_secret)"\s*:\s*)"[^"]*"/gi, '$1"[已脱敏]"'],
+  [/\beyJ[\w-]{6,}\.[\w-]{6,}\.[\w-]{6,}\b/g, "[已脱敏-JWT]"],
+  [/\b(?:sk|pk|ghp|gho|ghu|ghs|ghr|xox[baprs])-[\w-]{10,}\b/g, "[已脱敏-key]"],
+  // 裸 k=v（提问里常出现 token=…、password=…）：只掐足够长的值，避免把「token 数量」这类正常技术讨论也糊掉
+  [/\b(token|accessToken|refreshToken|api[_-]?key|apiKey|secret|password|cookie|client_secret|authorization)\s*[:=]\s*[\w./+-]{12,}/gi, "$1=[已脱敏]"],
+];
+
+/** 脱敏 + 按字节封顶截断（中文 3 字节/字，逐字累加才不会超预算）。 */
+export function maskText(v, cap = 100000) {
+  let s = String(v ?? "");
+  for (const [re, repl] of MASKS) s = s.replace(re, repl);
+  if (Buffer.byteLength(s, "utf8") <= cap) return s;
+  const tail = Buffer.from("\n…[已截断]", "utf8");
+  const budget = Math.max(0, cap - tail.length);
+  let bytes = 0, i = 0;
+  for (const ch of s) {
+    const n = Buffer.byteLength(ch, "utf8");
+    if (bytes + n > budget) break;
+    bytes += n;
+    i += ch.length;
+  }
+  return s.slice(0, i) + "\n…[已截断]";
+}
+
+function textOf(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) return content.map((p) => p?.text || (p?.image_url ? `[图片 ${String(p.image_url.url).slice(0, 60)}]` : "")).filter(Boolean).join("\n");
+  return "";
+}
+
+/** 只取「我发给大模型的问题」= 最后一条 user 消息（agent 回路每轮都带全量历史，全落会把环形窗口占满）。 */
+export function lastUserQuestion(body) {
+  const msgs = Array.isArray(body?.messages) ? body.messages : null;
+  if (msgs) { for (let i = msgs.length - 1; i >= 0; i--) if (msgs[i]?.role === "user") return textOf(msgs[i].content); }
+  const inp = body?.input; // Responses 端点形状兜底
+  if (typeof inp === "string") return inp;
+  if (Array.isArray(inp)) { for (let i = inp.length - 1; i >= 0; i--) if (inp[i]?.role === "user") return textOf(inp[i].content); }
+  return "";
+}
+
+const CUT_IN = "----8<----", CUT_OUT = "---->8----";
+
+function pushBlock(out, label, text, cap) {
+  const m = maskText(text, cap);
+  if (!m.trim()) return;
+  out.push(`[${label} · ${m.length} 字]`, CUT_IN, m, CUT_OUT, "");
+}
+
+/** 一条对话 = 一个可读块：头部一行元信息（ts= 是环形淘汰的唯一依据），正文三段。 */
+export function formatTalkEntry({ reqId, model, via, hops, status, stream, elapsedMs, question, thinking, answer, tools, usage, upstream, account, pick, ts } = {}) {
+  const now = Number.isFinite(ts) && ts > 0 ? ts : Date.now();
+  const head = [
+    `${MARK} ts=${now}`, `time=${fmtShanghaiYMDHMS(new Date(now))}`, `req=${reqId || "-"}`, `model=${model || "-"}`,
+    via ? `via=${via}` : "", hops != null ? `hops=${hops}` : "", `status=${status ?? "-"}`,
+    stream ? "stream=1" : "stream=0", Number.isFinite(elapsedMs) ? `elapsed=${Math.round(elapsedMs)}ms` : "",
+    usage ? `usage(prompt=${usage.prompt_tokens ?? "-"},completion=${usage.completion_tokens ?? "-"})` : "",
+    upstream ? `upstream=${upstream}` : "", account ? `account=${account}` : "", pick ? `pick=${pick}` : "",
+  ].filter(Boolean).join(" ");
+  const out = [head, ""];
+  pushBlock(out, "我问", question, 20000);
+  pushBlock(out, "思考", thinking, 100000);
+  pushBlock(out, "回答", answer, 200000);
+  const t = tools == null || tools === "" ? "" : String(tools);
+  pushBlock(out, "工具调用", t, 20000);
+  out.push("[END]", "");
+  return out.join("\n") + "\n";
+}
+
+/** 只读文件头 8KB 取最老一条的 ts（进程重启后重建内存基线，不必回读全文件）。 */
+function firstTs(file) {
+  let fd = null;
+  try {
+    fd = openSync(file, "r");
+    const buf = Buffer.alloc(8192);
+    const n = readSync(fd, buf, 0, 8192, 0);
+    const m = buf.subarray(0, n).toString("utf8").match(/^#talk-entry ts=(\d+)/m);
+    return m ? Number(m[1]) : null;
+  } catch { return null; }
+  finally { if (fd != null) { try { closeSync(fd); } catch {} } }
+}
+
+/** 追加一条（正常路径只 append；到窗口或超预算才整写淘汰）。 */
+export function appendTalkEntry({ file, block, atMs } = {}) {
+  try {
+    if (!file || !block) return 0;
+    mkdirSync(dirname(file), { recursive: true });
+    const isNew = !existsSync(file);
+    appendFileSync(file, block);
+    // mode 只在新建时生效；存量文件补一次 chmod（POSIX；Windows 无意义）
+    if (isNew) { try { chmodSync(file, 0o600); } catch {} }
+    const now = Number.isFinite(atMs) && atMs > 0 ? atMs : Date.now();
+    let base = oldestCache.get(file);
+    if (base == null) { base = firstTs(file); if (base == null) base = now; }
+    const size = (() => { try { return statSync(file).size; } catch { return 0; } })();
+    if (now - base > windowMs() || size > maxBytes()) trimTalkLog(file, now);
+    else oldestCache.set(file, Math.min(base, now));
+    return 1;
+  } catch { return 0; }
+}
+
+/** 按 `#talk-entry` 头行切块（内容行理论上可能撞前缀，故要求严格 ts= 形状才认头）。 */
+export function splitEntries(txt) {
+  const entries = [];
+  const lines = String(txt ?? "").split("\n");
+  let start = -1;
+  const close = (end) => {
+    if (start < 0) return;
+    const chunk = lines.slice(start, end);
+    const m = chunk[0].match(/^#talk-entry ts=(\d+)/);
+    entries.push({ ts: m ? Number(m[1]) : 0, text: chunk.join("\n") + "\n" });
+  };
+  for (let i = 0; i < lines.length; i++) {
+    if (/^#talk-entry ts=\d+/.test(lines[i])) { close(i); start = i; }
+  }
+  close(lines.length);
+  return entries;
+}
+
+/** 环形淘汰：先按时间窗口丢，再按字节预算从最老的丢（至少留最新 1 条）；tmp+rename 原子替换。 */
+export function trimTalkLog(file, nowTs = Date.now()) {
+  try {
+    if (!existsSync(file)) return 0;
+    const limit = nowTs - windowMs();
+    const cap = maxBytes();
+    const all = splitEntries(readFileSync(file, "utf8"));
+    // 多进程（restart 交接窗口）可能交错追加 → 按 ts 重排，保证「环形」语义与阅读顺序一致
+    let kept = all.filter((e) => e.ts > 0 && e.ts >= limit).sort((a, b) => a.ts - b.ts);
+    if (kept.length === 0) { rmSync(file, { force: true }); oldestCache.delete(file); return all.length; }
+    let bytes = Buffer.byteLength(kept.map((e) => e.text).join(""), "utf8");
+    while (bytes > cap && kept.length > 1) bytes -= Buffer.byteLength(kept.shift().text, "utf8");
+    const tmp = `${file}.${process.pid}.trim`;
+    writeFileSync(tmp, kept.map((e) => e.text).join(""), { mode: 0o600 });
+    renameSync(tmp, file);
+    oldestCache.set(file, kept[0].ts);
+    return all.length - kept.length;
+  } catch { return 0; }
+}
+
+/**
+ * relay 落盘的唯一入口：把 relay() 累积的 detail.talk（思考/正文/工具分片）拼成一条对话。
+ * 空轮（零正文零思考零工具）不占环形窗口 —— 那是 relay-pipeline 已经另行报错的场景。
+ */
+export function recordRelayTalk({ reqId, model, via, hops, body, out, echo = {} } = {}) {
+  if (!talkLogEnabled()) return false;
+  try {
+    const talk = out?.detail?.talk;
+    const answer = (talk?.content || []).join("");
+    const thinking = (talk?.reasoning || []).join("");
+    const tools = talk?.tools?.length ? JSON.stringify(talk.tools) : "";
+    const status = out?.status;
+    if (!answer && !thinking && !tools && Number(status) !== 200) return false;
+    if (!answer && !thinking && !tools) return false;
+    const ts = Date.now();
+    const block = formatTalkEntry({
+      reqId, model, via, hops, status, stream: Boolean(body?.stream),
+      elapsedMs: out?.totalMs, question: lastUserQuestion(body || {}),
+      thinking, answer, tools, usage: out?.detail?.usage, ...echo, ts,
+    });
+    appendTalkEntry({ file: talkLogFile(model), block, atMs: ts });
+    return true;
+  } catch { return false; }
+}
+
+/** 读最近 N 条（倒序，最新在前）—— 给 CLI/排障用。 */
+export function recentBlocks({ model, n = 10, file } = {}) {
+  try {
+    const target = file || talkLogFile(model);
+    if (!existsSync(target)) return [];
+    return splitEntries(readFileSync(target, "utf8")).map((e) => e.text).reverse().slice(0, n);
+  } catch { return []; }
+}
+
+/** 最近活跃的文件清单（哪些供应商×模型在这一小时里说过话）。 */
+export function listRecentFiles({ limit = 20 } = {}) {
+  try {
+    const dir = talkDir();
+    if (!existsSync(dir)) return [];
+    return readdirSync(dir, { withFileTypes: true })
+      .filter((d) => d.isFile() && d.name.endsWith("-talk.log"))
+      .map((d) => {
+        let mtime = 0;
+        try { mtime = statSync(join(dir, d.name)).mtimeMs; } catch {}
+        return { name: d.name, mtime, time: fmtShanghaiYMDHMS(new Date(mtime)) };
+      })
+      .sort((a, b) => b.mtime - a.mtime)
+      .slice(0, limit);
+  } catch { return []; }
+}

package/src/timeline.js

@@ -12,12 +12,15 @@ function outcome(status, reason) {
   return `${status}${reason ? ` ${short(reason)}` : ""}`;
 }
 
-export function formatTimeline({ reqId, model, direct = [], peers = [], retries = 0, result = null, totalMs = 0 } = {}) {
+ export function formatTimeline({ reqId, model, direct = [], peers = [], retries = 0, retryResult = null, result = null, totalMs = 0 } = {}) {
   const d = direct.length ? direct[direct.length - 1] : null;
   const ps = peers.map((p) => `[peer=${hostPort(p.peer)} ${p.ok ? "win" : "fail"} ${p.latencyMs != null ? `${p.latencyMs}ms` : "timing-"} ${short(p.message || p.status || "", 70)}]`);
   const r = result || {};
   const detail = r.detail || {};
   const res = r.status == null ? "-" : `${r.status}${detail.sawFinishReason ? ` ${detail.sawFinishReason}` : ""}${detail.toolCalls ? ` tools=${detail.toolCalls}` : ""}${detail.chars != null ? ` chars=${detail.chars}` : ""}${detail.interrupted ? " interrupted=1" : ""}${detail.timedOut ? " timedOut=1" : ""}`;
   const directText = d ? outcome(d.status, d.reason) : "-";
-  return `[req=${reqId || "-"}] [model=${model || "-"}] [direct=${directText}]${ps.length ? ` ${ps.join(" ")}` : ""}${retries ? ` [retry=${retries}]` : ""} [result=${res}] [total=${Math.max(0, Math.round(Number(totalMs) || 0))}ms]`;
+  // retry= 次数（保持既有 token 可 grep）；win=1 重试后救回，lost=1 重试用尽仍零正文。
+  const _rc = retryResult === "ok" ? " [retry_win=1]" : retryResult === "lost" ? " [retry_lost=1]" : "";
+  const retryTag = (retries || retryResult) ? ` [retry=${retries || 0}]${_rc}` : "";
+  return `[req=${reqId || "-"}] [model=${model || "-"}] [direct=${directText}]${ps.length ? ` ${ps.join(" ")}` : ""}${retryTag} [result=${res}] [total=${Math.max(0, Math.round(Number(totalMs) || 0))}ms]`;
 }

package/src/usage/record.js

@@ -78,7 +78,8 @@ export async function recordUsage(entry, { dir, now = new Date() } = {}) {
 
 // 从 relay 结果组装一行 usage —— 行形状由本模块拥有，调用方只交原始字段。
 // usage 即 metrics.js 的 extractUsageFromJson 输出（prompt/completion/total/reasoning）。
-export function recordChatUsage({ model, via, usage, ttfbMs, totalMs, tps, interrupted } = {}) {
+export function recordChatUsage({ model, via, usage, ttfbMs, totalMs, tps, interrupted, reasoningChars, stream } = {}) {
+  const rc = Number(reasoningChars);
   return recordUsage({
     model,
     via,
@@ -87,6 +88,12 @@ export function recordChatUsage({ model, via, usage, ttfbMs, totalMs, tps, inter
     completion_tokens: usage?.completion_tokens ?? 0,
     total_tokens: usage?.total_tokens ?? 0,
     reasoning_tokens: usage?.reasoning_tokens ?? 0,
+    // —— 以下为纯增字段：只存原始观测，chars/4 的估算留给聚合层（口径可演进，不必回填历史行）——
+    reasoning_chars: Number.isFinite(rc) && rc > 0 ? Math.trunc(rc) : 0,
+    // 上游是否明确上报过思考 tokens（metrics.js 只在数值有限时才写 reasoning_tokens）
+    reasoning_reported: Number.isFinite(usage?.reasoning_tokens) ? 1 : 0,
+    // 调用方没表态时整字段不落盘：「未知」≠「非流式」，覆盖度分母宁低不假高
+    ...(stream === undefined ? {} : { stream: stream ? 1 : 0 }),
     ttfbMs: Number.isFinite(ttfbMs) ? ttfbMs : null,
     totalMs: Number.isFinite(totalMs) ? totalMs : null,
     tps: Number.isFinite(tps) ? tps : null,

package/src/usage/report.js

@@ -16,6 +16,24 @@ function ms(v) {
   const n = Number(v);
   return Number.isFinite(n) && n >= 0 ? n : null;
 }
+// 思考 tokens 的四态判定（行级）：上报优先 → 有思考字符就估算 → 明确报 0 才算 0 → 否则未知。
+// 「上游报 0 却确有思考内容」按观测走估算：qoder 就是这形状，报 0 是上游没填，不是模型没想。
+export function resolveReasoning(row) {
+  const reportedVal = Number(row?.reasoning_tokens);
+  // 兼容判据：修复前的旧行没有 reasoning_reported，但带着真实上报值 —— 不能因缺字段退化成未知
+  const hasReported = row?.reasoning_reported === 1 || (Number.isFinite(reportedVal) && reportedVal > 0);
+  const chars = Number(row?.reasoning_chars);
+  const charCount = Number.isFinite(chars) && chars > 0 ? Math.trunc(chars) : 0;
+  if (hasReported && Number.isFinite(reportedVal) && reportedVal > 0) return { value: reportedVal, source: "reported" };
+  if (charCount > 0) {
+    const cap = Number(row?.completion_tokens);
+    const est = Math.ceil(charCount / 4); // 与 src/providers/cline/usage.js 的既有估算口径一致
+    return { value: Number.isFinite(cap) && cap > 0 ? Math.min(est, cap) : est, source: "estimated" };
+  }
+  if (hasReported) return { value: 0, source: "reported" }; // 该轮真的没思考
+  return { value: 0, source: "none" }; // 无从判断 → 表格渲染 —，绝不渲染成 0
+}
+
 
 function blank(id) {
   return {
@@ -31,6 +49,12 @@ function blank(id) {
     totalN: 0,
     completionMsSum: 0,
     tpsTokSum: 0,
+    reasoningReported: 0,
+    reasoningEstimated: 0,
+    reasoningReportedRows: 0,
+    reasoningEstimatedRows: 0,
+    reasoningChars: 0,
+    streamRequests: 0,
   };
 }
 
@@ -42,9 +66,16 @@ function finalize(a) {
     promptTokens: a.promptTokens,
     completionTokens: a.completionTokens,
     totalTokens: a.totalTokens,
-    reasoningTokens: a.reasoningTokens,
+    reasoningTokens: a.reasoningReported + a.reasoningEstimated,
     avgTtfbMs: a.ttfbN ? Math.round(a.ttfbSumMs / a.ttfbN) : null,
     avgTotalMs: a.totalN ? Math.round(a.totalSumMs / a.totalN) : null,
+    // 思考来源：上报与估算同时存在时是 mixed——合计值不冒充单一精确数（spec 锁死）
+    reasoningSource: a.reasoningReportedRows > 0 && a.reasoningEstimatedRows > 0 ? "mixed"
+      : a.reasoningReportedRows > 0 ? "reported"
+      : a.reasoningEstimatedRows > 0 ? "estimated" : "none",
+    reasoningChars: a.reasoningChars,
+    ttfSamples: a.ttfbN,
+    streamRequests: a.streamRequests,
     avgTps: a.completionMsSum > 0 ? Number((a.tpsTokSum / (a.completionMsSum / 1000)).toFixed(1)) : null,
   };
 }
@@ -57,7 +88,13 @@ function fold(acc, r) {
   acc.completionTokens += comp;
   const total = tok(r.total_tokens);
   acc.totalTokens += total || prompt + comp;
-  acc.reasoningTokens += tok(r.reasoning_tokens);
+  // 思考：四态分层累计（上报/估算各算各的，未知不贡献数值），原始字符数照实累加
+  const rs = resolveReasoning(r);
+  if (rs.source === "reported") { acc.reasoningReported += rs.value; acc.reasoningReportedRows++; }
+  else if (rs.source === "estimated") { acc.reasoningEstimated += rs.value; acc.reasoningEstimatedRows++; }
+  acc.reasoningChars += tok(r.reasoning_chars);
+  // 首字样本分母：非流式行既无首字也不该摊进分母；缺 stream 的旧行按未知保守计入
+  if (r.stream !== 0) acc.streamRequests++;
 
   const ttfb = ms(r.ttfbMs ?? r.ttfb_ms);
   if (ttfb != null) {