dsh-livebench-panel 0.2.2 → 0.2.4

package/lib/index.js

@@ -25,7 +25,7 @@
  * @module dsh-livebench-panel
  */
 import { spawn } from "node:child_process";
-import { readdirSync, readFileSync, existsSync, statSync, writeFileSync } from "node:fs";
+import { readdirSync, readFileSync, existsSync, statSync, writeFileSync, mkdirSync } from "node:fs";
 import { createRequire } from "node:module";
 import { delimiter, dirname, join } from "node:path";
 import { homedir } from "node:os";
@@ -365,7 +365,9 @@ function displayModelName(providerId, modelId) {
  */
 function writeGeneratedModelConfig(layout, YAML, { displayName, modelId, reasoningEffort, protocol }) {
 	const configDir = join(layout.livebenchDir, "model", "model_configs");
-	const target = join(configDir, "dsh_panel_generated.yaml");
+	// 每个模型一个独立配置文件：并发启动多模型时共享单文件会互相覆盖，
+	// 导致先启动的进程读到别人的配置 → anthropic/responses 路由失效 → 全 $ERROR$。
+	const target = join(configDir, "dsh_panel_generated__" + displayName.replace(/[^A-Za-z0-9._@-]/g, "_") + ".yaml");
 	void YAML;
 	const lines = [
 		"# Generated by dsh-livebench-panel — regenerated on every evaluation start.",
@@ -427,43 +429,103 @@ function readJsonl(path) {
 	}
 }
 
+/**
+ * 异步流式读取一个 JSONL 文件，对每行回调。
+ * 不整读入内存、不阻塞事件循环 —— coding 类 answer 单行可达数 MB，
+ * 同步整读会卡死 node 事件循环（表现为 DSH 断连假死）。
+ */
+async function streamJsonl(path, onRow) {
+	const fsPromises = await import("node:fs/promises");
+	let handle;
+	try {
+		handle = await fsPromises.open(path, "r");
+	} catch {
+		return;
+	}
+	try {
+		const decoder = new TextDecoder();
+		let buffer = "";
+		const chunkSize = 256 * 1024;
+		const buf = Buffer.alloc(chunkSize);
+		while (true) {
+			// 每读一块就让出事件循环，避免大文件连续占用
+			const { bytesRead } = await handle.read(buf, 0, chunkSize, null);
+			if (bytesRead === 0) break;
+			buffer += decoder.decode(buf.subarray(0, bytesRead), { stream: true });
+			let idx;
+			while ((idx = buffer.indexOf("\n")) >= 0) {
+				const line = buffer.slice(0, idx);
+				buffer = buffer.slice(idx + 1);
+				if (line.trim().length > 0) onRow(line);
+			}
+		}
+		buffer += decoder.decode();
+		if (buffer.trim().length > 0) onRow(buffer);
+	} catch {
+		/* 读取中断：尽力返回已读内容 */
+	} finally {
+		try { await handle.close(); } catch { /* ignore */ }
+	}
+}
+
+/**
+ * 从一行 answer JSON 提取 (model_id, 是否 $ERROR$)。不整对象解析时仍需 parse，
+ * 但只保留必要字段并立即释放 —— 避免大对象堆积。
+ */
+function parseAnswerLineLight(line) {
+	try {
+		const row = JSON.parse(line);
+		const turns = row.choices?.[0]?.turns;
+		const text = Array.isArray(turns) ? String(turns[0] ?? "") : "";
+		return { model: typeof row.model_id === "string" ? row.model_id : null, is_error: text.startsWith("$ERROR$"), raw: row };
+	} catch {
+		return null;
+	}
+}
+
 /**
  * Compute mean scores per (model, category, task) from ground-truth judgment
  * files, plus the judged/total question counts per task.
+ * 异步流式版：不再同步整读 answer 文件（coding 类单行数 MB，同步读会卡死
+ * node 事件循环导致 DSH 断连假死）。
  */
-function computeResults(dataDir) {
+async function computeResults(dataDir) {
 	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n, time}
 	const totals = new Map(); // task -> question count
 	const answers = new Map(); // `${model}\u0000${task}` -> {answered, errors}
+	const catByTask = new Map(); // task -> category（rows 阶段查分类用）
 	if (!existsSync(dataDir)) return { rows: [], taskTotals: {} };
 	for (const category of readDirSafe(dataDir)) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
 			const taskDir = join(catDir, task);
-			const questions = readJsonl(join(taskDir, "question.jsonl"));
-			totals.set(task, questions.length);
-			// answer side: how many answers each model produced, and how many of
-			// them were $ERROR$ (API call failures) — distinguishes "didn't run"
-			// from "ran and failed" from "ran and scored".
+			catByTask.set(task, category);
+			let questionCount = 0;
+			await streamJsonl(join(taskDir, "question.jsonl"), () => { questionCount += 1; });
+			totals.set(task, questionCount);
+			// answer 侧：各模型产出了多少答案、其中多少 $ERROR$（API 失败）
 			const answerDir = join(taskDir, "model_answer");
 			for (const file of readDirSafe(answerDir)) {
 				if (!file.endsWith(".jsonl")) continue;
-				for (const line of readJsonl(join(answerDir, file))) {
-					const model = typeof line.model_id === "string" ? line.model_id : file.slice(0, -".jsonl".length);
-					const turns = line.choices?.[0]?.turns;
-					const text = Array.isArray(turns) ? String(turns[0] ?? "") : "";
+				const fallbackModel = file.slice(0, -".jsonl".length);
+				await streamJsonl(join(answerDir, file), (line) => {
+					const parsed = parseAnswerLineLight(line);
+					if (!parsed) return;
+					const model = parsed.model ?? fallbackModel;
 					const key = `${model}\u0000${task}`;
 					const stat = answers.get(key) ?? { answered: 0, errors: 0 };
 					stat.answered += 1;
-					if (text.startsWith("$ERROR$")) stat.errors += 1;
+					if (parsed.is_error) stat.errors += 1;
 					answers.set(key, stat);
-				}
+				});
 			}
-			const judgments = readJsonl(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"));
-			for (const row of judgments) {
+			// judgment 侧：判分与时间戳
+			await streamJsonl(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), (line) => {
+				let row;
+				try { row = JSON.parse(line); } catch { return; }
 				const model = typeof row.model === "string" ? row.model : null;
 				const score = typeof row.score === "number" ? row.score : Number(row.score);
-				if (model === null || !Number.isFinite(score) || score < 0) continue;
+				if (model === null || !Number.isFinite(score) || score < 0) return;
 				const key = `${model}\u0000${task}`;
 				const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
 				entry.sum += score;
@@ -471,7 +533,9 @@ function computeResults(dataDir) {
 				const tstamp = Number(row.tstamp);
 				if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
 				judged.set(key, entry);
-			}
+			});
+			// 让出事件循环：任务之间穿插执行，保证 HTTP/websocket 实时响应
+			await new Promise((resolve) => setImmediate(resolve));
 		}
 	}
 	const keys = new Set([...judged.keys(), ...answers.keys()]);
@@ -479,15 +543,10 @@ function computeResults(dataDir) {
 		const entry = judged.get(key);
 		const stat = answers.get(key) ?? { answered: 0, errors: 0 };
 		const [model, task] = key.split("\u0000");
-		let category = entry?.category;
-		if (!category) {
-			for (const cat of readDirSafe(dataDir)) {
-				if (existsSync(join(dataDir, cat, task))) { category = cat; break; }
-			}
-		}
+		const category = entry?.category ?? catByTask.get(task) ?? "";
 		return {
 			model,
-			category: category ?? "",
+			category,
 			task,
 			score: entry && entry.n > 0 ? (entry.sum / entry.n) * 100 : null,
 			judged: entry?.n ?? 0,
@@ -501,6 +560,31 @@ function computeResults(dataDir) {
 	return { rows, taskTotals };
 }
 
+/**
+ * 异步统计某模型在指定 bench 范围内的 $ERROR$ 答案数（用于自动补跑判断）。
+ * 只扫 body.benchNames 涉及的任务目录，流式读取不阻塞事件循环。
+ */
+async function countErrorAnswersAsync(dataDir, displayName, body) {
+	if (!/^[A-Za-z0-9._@-]{1,160}$/.test(displayName)) return 0;
+	const benches = Array.isArray(body?.benchNames) ? body.benchNames : [];
+	let errors = 0;
+	for (const bn of benches) {
+		const parts = bn.split("/").filter((s) => s.length > 0); // live_bench/<cat>[/<task>]
+		if (parts.length < 2) continue;
+		const category = parts[1];
+		const tasks = parts.length >= 3 ? [parts[2]] : readDirSafe(join(dataDir, category));
+		for (const task of tasks) {
+			if (!/^[A-Za-z0-9_]{1,80}$/.test(task)) continue;
+			const file = join(dataDir, category, task, "model_answer", displayName + ".jsonl");
+			await streamJsonl(file, (line) => {
+				const parsed = parseAnswerLineLight(line);
+				if (parsed && parsed.is_error) errors += 1;
+			});
+		}
+	}
+	return errors;
+}
+
 /**
  * Remove every JSONL line of one model from a file (matched on the given
  * field), rewriting the file only when something changed.
@@ -740,18 +824,25 @@ function apply(ctx) {
 			record.exitCode = code === null ? -1 : code;
 			record.log.push(`[exit ${record.exitCode}]`);
 			// 自动补跑：正常结束但存在 $ERROR$ 答案时，用 --resume --retry-failures
-			// 只重跑失败题（最多 2 轮）。网络瞬断/超时导致的失败会被自动修复。
+			// 只重跑失败题（最多 2 轮）。统计改为异步流式（同步版曾阻塞事件循环，
+			// 且函数缺失会在 close 回调抛未捕获异常 → DSH 假死断连）。
 			if (record.exitCode === 0 && record.autoRetries < 2 && record.body) {
-				const errors = countErrorAnswers(layout.dataDir, record.displayName);
-				if (errors > 0) {
-					record.log.push(`[自动补跑] 检测到 ${errors} 条失败答案，自动重试（第 ${record.autoRetries + 1}/2 轮）`);
-					const retryBody = { ...record.body, resume: true, retryFailures: true };
-					setTimeout(() => {
-						startRun(retryBody, record.autoRetries + 1).catch((e) => {
-							appendLog(record, `[自动补跑启动失败] ${e.message}`);
-						});
-					}, 3000);
-				}
+				(async () => {
+					try {
+						const errors = await countErrorAnswersAsync(layout.dataDir, record.displayName, record.body);
+						if (errors > 0) {
+							record.log.push(`[自动补跑] 检测到 ${errors} 条失败答案，自动重试（第 ${record.autoRetries + 1}/2 轮）`);
+							const retryBody = { ...record.body, resume: true, retryFailures: true };
+							setTimeout(() => {
+								startRun(retryBody, record.autoRetries + 1).catch((e) => {
+									appendLog(record, `[自动补跑启动失败] ${e.message}`);
+								});
+							}, 3000);
+						}
+					} catch (e) {
+						record.log.push(`[自动补跑检查失败] ${e.message}`);
+					}
+				})();
 			}
 		});
 
@@ -920,7 +1011,7 @@ function apply(ctx) {
 				return;
 			}
 			const layout = livebenchLayout();
-			const { rows, taskTotals } = computeResults(layout.dataDir);
+			const { rows, taskTotals } = await computeResults(layout.dataDir);
 			sendJson(res, 200, { ok: true, rows, taskTotals });
 		},
 	}), `${name}: results route`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",