dsh-livebench-panel 0.2.11 → 0.2.12

package/lib/client.js

@@ -587,21 +587,28 @@ window.__ModuleLoader__.load({
 											h("td", null, model),
 											matrix.tasks.map((taskKeyCol) => {
 												const row = matrix.cells.get(`${model}__@__${taskKeyCol}`);
-												let cellText = "×";
-												let title = "该模型未执行此任务";
-												if (row) {
-													if (row.answered > 0 && row.errors >= row.answered) {
-														cellText = `E (${row.errors})`;
-														title = "该任务全部回答均 API 失败（$ERROR$），计 0 分";
-													} else if (row.judged === 0) {
-														cellText = "未判分";
-														title = "有答案但尚未判分";
-													} else {
-														cellText = `${row.score.toFixed(1)} (${row.judged}/${row.total})`;
-														if (row.errors > 0) title += ` · 其中 ${row.errors} 题 API 失败计 0 分`;
-													}
+												// 单元格：上方正确率，下方 (-没做/设定/总)。
+												// 没做/没做完（访问失败、网络中断）不计入正确率分母。
+												const subStyle = { fontSize: "10.5px", color: "var(--dsw-alias-label-tertiary)", fontWeight: "400" };
+												const sub = `(-${row ? (row.notDone ?? 0) : 0}/${row ? (row.configured ?? 0) : 0}/${row ? row.total : 0})`;
+												if (!row) {
+													return h("td", { key: taskKeyCol, className: c("score"), title: "该模型未执行此任务（无任何记录）" },
+														h("div", null, "×"),
+														h("div", { style: subStyle }, sub));
 												}
-												return h("td", { key: taskKeyCol, className: c("score"), title }, cellText);
+												const allFailed = row.answered > 0 && row.errors >= row.answered;
+												const title = allFailed
+													? `该任务 ${row.errors} 题全部访问失败（未做完），不计入正确率`
+													: `做完 ${row.done ?? 0} 题，没做/失败 ${row.notDone ?? 0} 题` +
+														(row.errors > 0 ? `（其中 ${row.errors} 题 API 失败）` : "");
+												if (allFailed || row.judged === 0) {
+													return h("td", { key: taskKeyCol, className: c("score"), title },
+														h("div", null, "—"),
+														h("div", { style: subStyle }, sub));
+												}
+												return h("td", { key: taskKeyCol, className: c("score"), title },
+													h("div", null, row.score.toFixed(1)),
+													h("div", { style: subStyle }, sub));
 											}),
 											h("td", null, h("button", {
 												className: c("btnGhost") + " " + c("btn"), title: "删除该模型的所有评测成绩",
@@ -658,7 +665,7 @@ window.__ModuleLoader__.load({
 								"提示：回答为 $ERROR$ 表示该题 API 调用失败（网络/鉴权/参数问题）计 0 分，可在上方日志区查看具体错误；zebra_puzzle（逻辑谜题）是公认最难的任务，低分属正常现象。"),
 						),
 						h("p", { className: c("hint") },
-							"评分读取 LiveBench 的 ground_truth_judgment.jsonl；正式榜单可用 release 2024-11-25。多选题集请在「分类/任务」中缩小范围，避免长时间运行。"),
+							"格式说明：单元格上方为正确率（= 做对 / 做完），下方括号 (-没做/设定/总)：没做或没做完的题（访问失败、网络中断、未跑到）不计入正确率。× 表示该模型完全没跑过该任务。正式榜单可用 release 2024-11-25。"),
 					);
 				}
 		/**

package/lib/index.js

@@ -307,6 +307,22 @@ function readDirSafe(dir) {
 	}
 }
 
+/** 列出目录下的【文件】（可按扩展名过滤）——注意 readDirSafe 只列目录。 */
+function readFilesSafe(dir, ext) {
+	try {
+		return readdirSync(dir).filter((f) => {
+			try {
+				if (!statSync(join(dir, f)).isFile()) return false;
+				return ext === undefined || f.endsWith(ext);
+			} catch {
+				return false;
+			}
+		});
+	} catch {
+		return [];
+	}
+}
+
 function readdirNames(dir) {
 	try {
 		return readdirSync(dir).filter((entry) => {
@@ -506,11 +522,22 @@ async function readTextFileAsync(path) {
 async function computeResults(dataDir) {
 	const now = Date.now();
 	if (resultsCache.data !== null && now - resultsCache.at < 5000) return resultsCache.data;
-	const judged = new Map(); // `${model}${task}` -> {model, category, task, sum, n, time}
+	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n, time}
 	const totals = new Map(); // task -> question count
-	const answers = new Map(); // `${model}${task}` -> {answered, errors}
+	const answers = new Map(); // `${model}\u0000${task}` -> {answered, errors}
 	const catByTask = new Map();
 	if (!existsSync(dataDir)) return { rows: [], taskTotals: {} };
+	// 读取每次评测的配置范围（displayName -> {benchNames, begin, end}）
+	const runMeta = new Map();
+	try {
+		const metaDir = join(dataDir, ".dsh_runs");
+		for (const f of readdirSync(metaDir)) {
+			if (!f.endsWith(".json")) continue;
+			try {
+				runMeta.set(f.slice(0, -5), JSON.parse(readFileSync(join(metaDir, f), "utf8")));
+			} catch { /* 单个元数据损坏忽略 */ }
+		}
+	} catch { /* 无元数据目录（历史数据） */ }
 	for (const category of readDirSafe(dataDir)) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
@@ -519,12 +546,11 @@ async function computeResults(dataDir) {
 			const qText = await readTextFileAsync(join(taskDir, "question.jsonl"));
 			totals.set(task, qText === null ? 0 : qText.split("\n").reduce((n, l) => n + (l.trim().length > 0 ? 1 : 0), 0));
 			const answerDir = join(taskDir, "model_answer");
-			for (const file of readDirSafe(answerDir)) {
-				if (!file.endsWith(".jsonl")) continue;
+			for (const file of readFilesSafe(answerDir, ".jsonl")) {
 				const model = file.slice(0, -".jsonl".length);
 				const text = await readTextFileAsync(join(answerDir, file));
 				if (text === null) continue;
-				const key = `${model}${task}`;
+				const key = `${model}\u0000${task}`;
 				const stat = answers.get(key) ?? { answered: 0, errors: 0 };
 				for (const line of text.split("\n")) {
 					if (line.trim().length === 0) continue;
@@ -542,7 +568,7 @@ async function computeResults(dataDir) {
 					const model = typeof row.model === "string" ? row.model : null;
 					const score = typeof row.score === "number" ? row.score : Number(row.score);
 					if (model === null || !Number.isFinite(score) || score < 0) continue;
-					const key = `${model}${task}`;
+					const key = `${model}\u0000${task}`;
 					const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
 					entry.sum += score;
 					entry.n += 1;
@@ -559,18 +585,45 @@ async function computeResults(dataDir) {
 	const rows = [...keys].map((key) => {
 		const entry = judged.get(key);
 		const stat = answers.get(key) ?? { answered: 0, errors: 0 };
-		const [model, task] = key.split("");
+		const [model, task] = key.split("\u0000");
 		const category = entry?.category ?? catByTask.get(task) ?? "";
+		const total = totals.get(task) ?? 0;
+		const judgedCount = entry?.n ?? 0;
+		// 做完的题 = 实际产出的有效答案（尝试数 - 访问失败数）
+		const done = Math.max(0, stat.answered - stat.errors);
+		// 设定题数：优先用评测元数据的范围（end 含端点），否则用尝试/判分中的较大值
+		let configured = Math.max(stat.answered, judgedCount);
+		const meta = runMeta.get(model);
+		if (meta !== undefined) {
+			const inScope = (meta.benchNames || []).some((bn) => {
+				const parts = String(bn).split("/").filter((s) => s.length > 0);
+				if (parts.length >= 3) return parts[1] === category && parts[2] === task;
+				if (parts.length === 2) return parts[1] === category;
+				return true;
+			});
+			if (inScope) {
+				const hasRange = meta.begin !== null && meta.begin !== undefined && meta.end !== null && meta.end !== undefined;
+				configured = hasRange ? Math.min(total, Number(meta.end) - Number(meta.begin) + 1) : total;
+			}
+		}
+		if (configured < stat.answered) configured = stat.answered;
+		const notDone = Math.max(0, configured - done);
+		// 正确率 = 做对的题 / 已判分的做完题（访问失败/未做的题不计入分母）
+		const judgedDone = Math.max(0, judgedCount - stat.errors);
+		const score = entry && judgedDone > 0 ? (entry.sum / judgedDone) * 100 : null;
 		return {
 			model,
 			category,
 			task,
-			score: entry && entry.n > 0 ? (entry.sum / entry.n) * 100 : null,
+			score,
 			judged: entry?.n ?? 0,
-			total: totals.get(task) ?? 0,
+			total,
 			time: entry && entry.time > 0 ? entry.time : null,
 			answered: stat.answered,
 			errors: stat.errors,
+			done,
+			configured,
+			notDone,
 		};
 	});
 	const result = { rows, taskTotals: Object.fromEntries(totals) };
@@ -808,6 +861,19 @@ function apply(ctx) {
 			}
 		}
 
+		// 记录本次评测的配置范围：/results 用它计算每个任务的"设定题数"
+		// （未做/失败的题数 = 设定题数 - 实际产出的有效答案数）。
+		try {
+			const metaDir = join(layout.dataDir, ".dsh_runs");
+			if (!existsSync(metaDir)) mkdirSync(metaDir, { recursive: true });
+			writeFileSync(join(metaDir, displayName + ".json"), JSON.stringify({
+				benchNames: benchNames !== null ? benchNames : [benchParts.join("/")],
+				begin: body.begin ?? null,
+				end: body.end ?? null,
+				startedAt: new Date().toISOString(),
+			}), "utf8");
+		} catch { /* 元数据写入失败不影响评测 */ }
+
 		const child = spawn(layout.pythonExe, args, {
 			cwd: layout.livebenchDir,
 			env,
@@ -865,7 +931,7 @@ function apply(ctx) {
 								startRun(retryBody, record.autoRetries + 1).catch((e) => {
 									appendLog(record, `[自动补跑启动失败] ${e.message}`);
 								});
-							}, 15000);
+							}, [15000, 60000, 180000][Math.min(record.autoRetries, 2)]);
 						}
 					} catch (e) {
 						record.log.push(`[自动补跑检查失败] ${e.message}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",