dsh-livebench-panel 0.2.10 → 0.2.11

package/lib/index.js

@@ -485,56 +485,73 @@ function parseAnswerLineLight(line) {
 
 /**
  * Compute mean scores per (model, category, task) from ground-truth judgment
- * files, plus the judged/total question counts per task.
- * 异步流式版：不再同步整读 answer 文件（coding 类单行数 MB，同步读会卡死
- * node 事件循环导致 DSH 断连假死）。
+ * files, plus judged/total question counts per task.
+ *
+ * 性能：数据目录可能上百 MB（每次评测生成独立答案文件）。这里：
+ *  - answer 文件用【文件名】当模型名，只做行数与 "$ERROR$" 子串统计 —— 不解析大 JSON 行；
+ *  - question.jsonl 只数行数；
+ *  - 结果缓存 5 秒，避免前端频繁刷新时重复全量扫描。
  */
+let resultsCache = { at: 0, data: null };
+
+async function readTextFileAsync(path) {
+	const fsPromises = await import("node:fs/promises");
+	try {
+		return await fsPromises.readFile(path, "utf8");
+	} catch {
+		return null;
+	}
+}
+
 async function computeResults(dataDir) {
-	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n, time}
+	const now = Date.now();
+	if (resultsCache.data !== null && now - resultsCache.at < 5000) return resultsCache.data;
+	const judged = new Map(); // `${model}${task}` -> {model, category, task, sum, n, time}
 	const totals = new Map(); // task -> question count
-	const answers = new Map(); // `${model}\u0000${task}` -> {answered, errors}
-	const catByTask = new Map(); // task -> category（rows 阶段查分类用）
+	const answers = new Map(); // `${model}${task}` -> {answered, errors}
+	const catByTask = new Map();
 	if (!existsSync(dataDir)) return { rows: [], taskTotals: {} };
 	for (const category of readDirSafe(dataDir)) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
 			const taskDir = join(catDir, task);
 			catByTask.set(task, category);
-			let questionCount = 0;
-			await streamJsonl(join(taskDir, "question.jsonl"), () => { questionCount += 1; });
-			totals.set(task, questionCount);
-			// answer 侧：各模型产出了多少答案、其中多少 $ERROR$（API 失败）
+			const qText = await readTextFileAsync(join(taskDir, "question.jsonl"));
+			totals.set(task, qText === null ? 0 : qText.split("\n").reduce((n, l) => n + (l.trim().length > 0 ? 1 : 0), 0));
 			const answerDir = join(taskDir, "model_answer");
 			for (const file of readDirSafe(answerDir)) {
 				if (!file.endsWith(".jsonl")) continue;
-				const fallbackModel = file.slice(0, -".jsonl".length);
-				await streamJsonl(join(answerDir, file), (line) => {
-					const parsed = parseAnswerLineLight(line);
-					if (!parsed) return;
-					const model = parsed.model ?? fallbackModel;
-					const key = `${model}\u0000${task}`;
-					const stat = answers.get(key) ?? { answered: 0, errors: 0 };
+				const model = file.slice(0, -".jsonl".length);
+				const text = await readTextFileAsync(join(answerDir, file));
+				if (text === null) continue;
+				const key = `${model}${task}`;
+				const stat = answers.get(key) ?? { answered: 0, errors: 0 };
+				for (const line of text.split("\n")) {
+					if (line.trim().length === 0) continue;
 					stat.answered += 1;
-					if (parsed.is_error) stat.errors += 1;
-					answers.set(key, stat);
-				});
+					if (line.includes('"$ERROR$"')) stat.errors += 1;
+				}
+				answers.set(key, stat);
 			}
-			// judgment 侧：判分与时间戳
-			await streamJsonl(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), (line) => {
-				let row;
-				try { row = JSON.parse(line); } catch { return; }
-				const model = typeof row.model === "string" ? row.model : null;
-				const score = typeof row.score === "number" ? row.score : Number(row.score);
-				if (model === null || !Number.isFinite(score) || score < 0) return;
-				const key = `${model}\u0000${task}`;
-				const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
-				entry.sum += score;
-				entry.n += 1;
-				const tstamp = Number(row.tstamp);
-				if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
-				judged.set(key, entry);
-			});
-			// 让出事件循环：任务之间穿插执行，保证 HTTP/websocket 实时响应
+			const jText = await readTextFileAsync(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"));
+			if (jText !== null) {
+				for (const line of jText.split("\n")) {
+					if (line.trim().length === 0) continue;
+					let row;
+					try { row = JSON.parse(line); } catch { continue; }
+					const model = typeof row.model === "string" ? row.model : null;
+					const score = typeof row.score === "number" ? row.score : Number(row.score);
+					if (model === null || !Number.isFinite(score) || score < 0) continue;
+					const key = `${model}${task}`;
+					const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
+					entry.sum += score;
+					entry.n += 1;
+					const tstamp = Number(row.tstamp);
+					if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
+					judged.set(key, entry);
+				}
+			}
+			// 让出事件循环，保证 HTTP/websocket 实时响应
 			await new Promise((resolve) => setImmediate(resolve));
 		}
 	}
@@ -542,7 +559,7 @@ async function computeResults(dataDir) {
 	const rows = [...keys].map((key) => {
 		const entry = judged.get(key);
 		const stat = answers.get(key) ?? { answered: 0, errors: 0 };
-		const [model, task] = key.split("\u0000");
+		const [model, task] = key.split("");
 		const category = entry?.category ?? catByTask.get(task) ?? "";
 		return {
 			model,
@@ -556,8 +573,9 @@ async function computeResults(dataDir) {
 			errors: stat.errors,
 		};
 	});
-	const taskTotals = Object.fromEntries(totals);
-	return { rows, taskTotals };
+	const result = { rows, taskTotals: Object.fromEntries(totals) };
+	resultsCache = { at: Date.now(), data: result };
+	return result;
 }
 
 /**
@@ -1101,6 +1119,7 @@ function apply(ctx) {
 				removed += filterJsonlByModel(join(taskDir, "model_answer", `${model}.jsonl`), "model_id", model);
 				removed += filterJsonlByModel(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), "model", model);
 			}
+			resultsCache = { at: 0, data: null };
 			sendJson(res, 200, { ok: true, removed });
 		},
 	}), `${name}: delete route`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.2.10",
+  "version": "0.2.11",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",