dsh-livebench-panel 0.2.10 → 0.2.12

package/lib/client.js

@@ -587,21 +587,28 @@ window.__ModuleLoader__.load({
 											h("td", null, model),
 											matrix.tasks.map((taskKeyCol) => {
 												const row = matrix.cells.get(`${model}__@__${taskKeyCol}`);
-												let cellText = "×";
-												let title = "该模型未执行此任务";
-												if (row) {
-													if (row.answered > 0 && row.errors >= row.answered) {
-														cellText = `E (${row.errors})`;
-														title = "该任务全部回答均 API 失败（$ERROR$），计 0 分";
-													} else if (row.judged === 0) {
-														cellText = "未判分";
-														title = "有答案但尚未判分";
-													} else {
-														cellText = `${row.score.toFixed(1)} (${row.judged}/${row.total})`;
-														if (row.errors > 0) title += ` · 其中 ${row.errors} 题 API 失败计 0 分`;
-													}
+												// 单元格：上方正确率，下方 (-没做/设定/总)。
+												// 没做/没做完（访问失败、网络中断）不计入正确率分母。
+												const subStyle = { fontSize: "10.5px", color: "var(--dsw-alias-label-tertiary)", fontWeight: "400" };
+												const sub = `(-${row ? (row.notDone ?? 0) : 0}/${row ? (row.configured ?? 0) : 0}/${row ? row.total : 0})`;
+												if (!row) {
+													return h("td", { key: taskKeyCol, className: c("score"), title: "该模型未执行此任务（无任何记录）" },
+														h("div", null, "×"),
+														h("div", { style: subStyle }, sub));
 												}
-												return h("td", { key: taskKeyCol, className: c("score"), title }, cellText);
+												const allFailed = row.answered > 0 && row.errors >= row.answered;
+												const title = allFailed
+													? `该任务 ${row.errors} 题全部访问失败（未做完），不计入正确率`
+													: `做完 ${row.done ?? 0} 题，没做/失败 ${row.notDone ?? 0} 题` +
+														(row.errors > 0 ? `（其中 ${row.errors} 题 API 失败）` : "");
+												if (allFailed || row.judged === 0) {
+													return h("td", { key: taskKeyCol, className: c("score"), title },
+														h("div", null, "—"),
+														h("div", { style: subStyle }, sub));
+												}
+												return h("td", { key: taskKeyCol, className: c("score"), title },
+													h("div", null, row.score.toFixed(1)),
+													h("div", { style: subStyle }, sub));
 											}),
 											h("td", null, h("button", {
 												className: c("btnGhost") + " " + c("btn"), title: "删除该模型的所有评测成绩",
@@ -658,7 +665,7 @@ window.__ModuleLoader__.load({
 								"提示：回答为 $ERROR$ 表示该题 API 调用失败（网络/鉴权/参数问题）计 0 分，可在上方日志区查看具体错误；zebra_puzzle（逻辑谜题）是公认最难的任务，低分属正常现象。"),
 						),
 						h("p", { className: c("hint") },
-							"评分读取 LiveBench 的 ground_truth_judgment.jsonl；正式榜单可用 release 2024-11-25。多选题集请在「分类/任务」中缩小范围，避免长时间运行。"),
+							"格式说明：单元格上方为正确率（= 做对 / 做完），下方括号 (-没做/设定/总)：没做或没做完的题（访问失败、网络中断、未跑到）不计入正确率。× 表示该模型完全没跑过该任务。正式榜单可用 release 2024-11-25。"),
 					);
 				}
 		/**

package/lib/index.js

@@ -307,6 +307,22 @@ function readDirSafe(dir) {
 	}
 }
 
+/** 列出目录下的【文件】（可按扩展名过滤）——注意 readDirSafe 只列目录。 */
+function readFilesSafe(dir, ext) {
+	try {
+		return readdirSync(dir).filter((f) => {
+			try {
+				if (!statSync(join(dir, f)).isFile()) return false;
+				return ext === undefined || f.endsWith(ext);
+			} catch {
+				return false;
+			}
+		});
+	} catch {
+		return [];
+	}
+}
+
 function readdirNames(dir) {
 	try {
 		return readdirSync(dir).filter((entry) => {
@@ -485,56 +501,83 @@ function parseAnswerLineLight(line) {
 
 /**
  * Compute mean scores per (model, category, task) from ground-truth judgment
- * files, plus the judged/total question counts per task.
- * 异步流式版：不再同步整读 answer 文件（coding 类单行数 MB，同步读会卡死
- * node 事件循环导致 DSH 断连假死）。
+ * files, plus judged/total question counts per task.
+ *
+ * 性能：数据目录可能上百 MB（每次评测生成独立答案文件）。这里：
+ *  - answer 文件用【文件名】当模型名，只做行数与 "$ERROR$" 子串统计 —— 不解析大 JSON 行；
+ *  - question.jsonl 只数行数；
+ *  - 结果缓存 5 秒，避免前端频繁刷新时重复全量扫描。
  */
+let resultsCache = { at: 0, data: null };
+
+async function readTextFileAsync(path) {
+	const fsPromises = await import("node:fs/promises");
+	try {
+		return await fsPromises.readFile(path, "utf8");
+	} catch {
+		return null;
+	}
+}
+
 async function computeResults(dataDir) {
+	const now = Date.now();
+	if (resultsCache.data !== null && now - resultsCache.at < 5000) return resultsCache.data;
 	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n, time}
 	const totals = new Map(); // task -> question count
 	const answers = new Map(); // `${model}\u0000${task}` -> {answered, errors}
-	const catByTask = new Map(); // task -> category（rows 阶段查分类用）
+	const catByTask = new Map();
 	if (!existsSync(dataDir)) return { rows: [], taskTotals: {} };
+	// 读取每次评测的配置范围（displayName -> {benchNames, begin, end}）
+	const runMeta = new Map();
+	try {
+		const metaDir = join(dataDir, ".dsh_runs");
+		for (const f of readdirSync(metaDir)) {
+			if (!f.endsWith(".json")) continue;
+			try {
+				runMeta.set(f.slice(0, -5), JSON.parse(readFileSync(join(metaDir, f), "utf8")));
+			} catch { /* 单个元数据损坏忽略 */ }
+		}
+	} catch { /* 无元数据目录（历史数据） */ }
 	for (const category of readDirSafe(dataDir)) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
 			const taskDir = join(catDir, task);
 			catByTask.set(task, category);
-			let questionCount = 0;
-			await streamJsonl(join(taskDir, "question.jsonl"), () => { questionCount += 1; });
-			totals.set(task, questionCount);
-			// answer 侧：各模型产出了多少答案、其中多少 $ERROR$（API 失败）
+			const qText = await readTextFileAsync(join(taskDir, "question.jsonl"));
+			totals.set(task, qText === null ? 0 : qText.split("\n").reduce((n, l) => n + (l.trim().length > 0 ? 1 : 0), 0));
 			const answerDir = join(taskDir, "model_answer");
-			for (const file of readDirSafe(answerDir)) {
-				if (!file.endsWith(".jsonl")) continue;
-				const fallbackModel = file.slice(0, -".jsonl".length);
-				await streamJsonl(join(answerDir, file), (line) => {
-					const parsed = parseAnswerLineLight(line);
-					if (!parsed) return;
-					const model = parsed.model ?? fallbackModel;
-					const key = `${model}\u0000${task}`;
-					const stat = answers.get(key) ?? { answered: 0, errors: 0 };
+			for (const file of readFilesSafe(answerDir, ".jsonl")) {
+				const model = file.slice(0, -".jsonl".length);
+				const text = await readTextFileAsync(join(answerDir, file));
+				if (text === null) continue;
+				const key = `${model}\u0000${task}`;
+				const stat = answers.get(key) ?? { answered: 0, errors: 0 };
+				for (const line of text.split("\n")) {
+					if (line.trim().length === 0) continue;
 					stat.answered += 1;
-					if (parsed.is_error) stat.errors += 1;
-					answers.set(key, stat);
-				});
+					if (line.includes('"$ERROR$"')) stat.errors += 1;
+				}
+				answers.set(key, stat);
 			}
-			// judgment 侧：判分与时间戳
-			await streamJsonl(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), (line) => {
-				let row;
-				try { row = JSON.parse(line); } catch { return; }
-				const model = typeof row.model === "string" ? row.model : null;
-				const score = typeof row.score === "number" ? row.score : Number(row.score);
-				if (model === null || !Number.isFinite(score) || score < 0) return;
-				const key = `${model}\u0000${task}`;
-				const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
-				entry.sum += score;
-				entry.n += 1;
-				const tstamp = Number(row.tstamp);
-				if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
-				judged.set(key, entry);
-			});
-			// 让出事件循环：任务之间穿插执行，保证 HTTP/websocket 实时响应
+			const jText = await readTextFileAsync(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"));
+			if (jText !== null) {
+				for (const line of jText.split("\n")) {
+					if (line.trim().length === 0) continue;
+					let row;
+					try { row = JSON.parse(line); } catch { continue; }
+					const model = typeof row.model === "string" ? row.model : null;
+					const score = typeof row.score === "number" ? row.score : Number(row.score);
+					if (model === null || !Number.isFinite(score) || score < 0) continue;
+					const key = `${model}\u0000${task}`;
+					const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
+					entry.sum += score;
+					entry.n += 1;
+					const tstamp = Number(row.tstamp);
+					if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
+					judged.set(key, entry);
+				}
+			}
+			// 让出事件循环，保证 HTTP/websocket 实时响应
 			await new Promise((resolve) => setImmediate(resolve));
 		}
 	}
@@ -544,20 +587,48 @@ async function computeResults(dataDir) {
 		const stat = answers.get(key) ?? { answered: 0, errors: 0 };
 		const [model, task] = key.split("\u0000");
 		const category = entry?.category ?? catByTask.get(task) ?? "";
+		const total = totals.get(task) ?? 0;
+		const judgedCount = entry?.n ?? 0;
+		// 做完的题 = 实际产出的有效答案（尝试数 - 访问失败数）
+		const done = Math.max(0, stat.answered - stat.errors);
+		// 设定题数：优先用评测元数据的范围（end 含端点），否则用尝试/判分中的较大值
+		let configured = Math.max(stat.answered, judgedCount);
+		const meta = runMeta.get(model);
+		if (meta !== undefined) {
+			const inScope = (meta.benchNames || []).some((bn) => {
+				const parts = String(bn).split("/").filter((s) => s.length > 0);
+				if (parts.length >= 3) return parts[1] === category && parts[2] === task;
+				if (parts.length === 2) return parts[1] === category;
+				return true;
+			});
+			if (inScope) {
+				const hasRange = meta.begin !== null && meta.begin !== undefined && meta.end !== null && meta.end !== undefined;
+				configured = hasRange ? Math.min(total, Number(meta.end) - Number(meta.begin) + 1) : total;
+			}
+		}
+		if (configured < stat.answered) configured = stat.answered;
+		const notDone = Math.max(0, configured - done);
+		// 正确率 = 做对的题 / 已判分的做完题（访问失败/未做的题不计入分母）
+		const judgedDone = Math.max(0, judgedCount - stat.errors);
+		const score = entry && judgedDone > 0 ? (entry.sum / judgedDone) * 100 : null;
 		return {
 			model,
 			category,
 			task,
-			score: entry && entry.n > 0 ? (entry.sum / entry.n) * 100 : null,
+			score,
 			judged: entry?.n ?? 0,
-			total: totals.get(task) ?? 0,
+			total,
 			time: entry && entry.time > 0 ? entry.time : null,
 			answered: stat.answered,
 			errors: stat.errors,
+			done,
+			configured,
+			notDone,
 		};
 	});
-	const taskTotals = Object.fromEntries(totals);
-	return { rows, taskTotals };
+	const result = { rows, taskTotals: Object.fromEntries(totals) };
+	resultsCache = { at: Date.now(), data: result };
+	return result;
 }
 
 /**
@@ -790,6 +861,19 @@ function apply(ctx) {
 			}
 		}
 
+		// 记录本次评测的配置范围：/results 用它计算每个任务的"设定题数"
+		// （未做/失败的题数 = 设定题数 - 实际产出的有效答案数）。
+		try {
+			const metaDir = join(layout.dataDir, ".dsh_runs");
+			if (!existsSync(metaDir)) mkdirSync(metaDir, { recursive: true });
+			writeFileSync(join(metaDir, displayName + ".json"), JSON.stringify({
+				benchNames: benchNames !== null ? benchNames : [benchParts.join("/")],
+				begin: body.begin ?? null,
+				end: body.end ?? null,
+				startedAt: new Date().toISOString(),
+			}), "utf8");
+		} catch { /* 元数据写入失败不影响评测 */ }
+
 		const child = spawn(layout.pythonExe, args, {
 			cwd: layout.livebenchDir,
 			env,
@@ -847,7 +931,7 @@ function apply(ctx) {
 								startRun(retryBody, record.autoRetries + 1).catch((e) => {
 									appendLog(record, `[自动补跑启动失败] ${e.message}`);
 								});
-							}, 15000);
+							}, [15000, 60000, 180000][Math.min(record.autoRetries, 2)]);
 						}
 					} catch (e) {
 						record.log.push(`[自动补跑检查失败] ${e.message}`);
@@ -1101,6 +1185,7 @@ function apply(ctx) {
 				removed += filterJsonlByModel(join(taskDir, "model_answer", `${model}.jsonl`), "model_id", model);
 				removed += filterJsonlByModel(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), "model", model);
 			}
+			resultsCache = { at: 0, data: null };
 			sendJson(res, 200, { ok: true, removed });
 		},
 	}), `${name}: delete route`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",