dsh-livebench-panel 0.2.21 → 0.2.22

package/lib/client.js

@@ -805,10 +805,14 @@ window.__ModuleLoader__.load({
 														row.errors > 0 ? `${row.errors} 题 API 失败` : null,
 														emptyCount > 0 ? `${emptyCount} 题思考占满 max-tokens、未产出答案` : null,
 													].filter(Boolean).join("，");
+													const missing = Array.isArray(row.missingIndexes) ? row.missingIndexes : null;
+													const missingText = missing !== null && missing.length > 0
+														? `\n没做出来的题号（0 起，可复制去重跑）：${missing.join("、")}`
+														: "";
 													const title = basePrefix + (allFailed
 														? `本次选择 ${row.configured ?? 0} 题，全部没做出来（${reasons}），不计入正确率 ${sub}`
 														: `本次选择 ${row.configured ?? 0} 题，做出来 ${row.done ?? 0} 题，没做出来 ${row.notDone ?? 0} 题 ${sub}`
-															+ (reasons ? `（其中 ${reasons}）` : ""));													if (allFailed || row.judged === 0) {
+															+ (reasons ? `（其中 ${reasons}）` : "")) + missingText;													if (allFailed || row.judged === 0) {
 														return h("td", { key: taskKeyCol, className: cls, title },
 															h("div", { className: c("dim") }, "—"),
 															h("div", { className: c("cellSub") }, sub));

package/lib/index.js

@@ -627,6 +627,33 @@ function isEmptyAnswerLine(line) {
 	return EMPTY_ANSWER_RE.test(line);
 }
 
+/** 从答案行里抽 question_id（只做正则，不 JSON.parse）。 */
+const QUESTION_ID_RE = /"question_id"\s*:\s*"([A-Za-z0-9_.:-]{8,120})"/;
+
+/**
+ * Baseline 评测里"没做出来的题号"。
+ *
+ * 编号是 **该题库内的 0 起下标**，与「题目序号范围」的编号一致，所以可以直接复制去重跑。
+ * 范围先按题库实际题数裁剪 —— 「0-1000」这种越界输入不会把编号算歪（基准永远是实际题数）。
+ *
+ * @param {object|undefined} meta 运行元数据（需要 baselineIds / begin / end）
+ * @param {Set<string>} seen 答案文件里实际出现的 question_id
+ * @returns {number[]|null} 不是 baseline 运行时返回 null
+ */
+function missingBaselineIndexes(meta, seen) {
+	if (!meta || !Array.isArray(meta.baselineIds) || meta.baselineIds.length === 0) return null;
+	const ids = meta.baselineIds;
+	const hasBegin = meta.begin !== null && meta.begin !== undefined;
+	const hasEnd = meta.end !== null && meta.end !== undefined;
+	const from = hasBegin ? Math.max(0, Math.min(ids.length, Number(meta.begin))) : 0;
+	const to = hasEnd ? Math.max(0, Math.min(ids.length, Number(meta.end) + 1)) : ids.length;
+	const out = [];
+	for (let i = from; i < Math.max(from, to); i += 1) {
+		if (!seen.has(ids[i])) out.push(i);
+	}
+	return out;
+}
+
 async function readTextFileAsync(path) {
 	const fsPromises = await import("node:fs/promises");
 	try {
@@ -668,12 +695,21 @@ async function computeResults(dataDir) {
 				const text = await readTextFileAsync(join(answerDir, file));
 				if (text === null) continue;
 				const key = `${model}\u0000${task}`;
-				const stat = answers.get(key) ?? { answered: 0, errors: 0, empty: 0 };
+				const stat = answers.get(key) ?? { answered: 0, errors: 0, empty: 0, seen: null };
+				// baseline 评测要回答"少做了哪几个题"，所以需要收集实际出现的 question_id。
+				// 只用正则抽 id（不 JSON.parse），对几百 KB～几 MB 的答案文件也够快。
+				const runMetaEntry = runMeta.get(model);
+				const trackIds = runMetaEntry && Array.isArray(runMetaEntry.baselineIds) && runMetaEntry.baselineIds.length > 0;
+				if (trackIds && stat.seen === null) stat.seen = new Set();
 				for (const line of text.split("\n")) {
 					if (line.trim().length === 0) continue;
 					stat.answered += 1;
 					if (line.includes('"$ERROR$"')) stat.errors += 1;
 					else if (isEmptyAnswerLine(line)) stat.empty += 1;
+					if (trackIds) {
+						const idMatch = line.match(QUESTION_ID_RE);
+						if (idMatch !== null) stat.seen.add(idMatch[1]);
+					}
 				}
 				// 空文件（尚未产出答案，或成绩已被删除只留下 0 字节壳）不计入结果：
 				// 否则删掉的记录会在下一次扫描时凭“文件名仍在”重新冒出来。
@@ -753,6 +789,9 @@ async function computeResults(dataDir) {
 		// 正确率 = 做对的题 / **做出来的题**（API 失败、思考吃满 token 没产出答案的都不进分母）
 		const judgedDone = Math.max(0, judgedCount - notProduced);
 		const score = entry && judgedDone > 0 ? (entry.sum / judgedDone) * 100 : null;
+		// Baseline 评测：算出**具体哪几个题号没做出来**（题号 = 该题库内的 0 起下标，
+		// 与「题目序号范围」的编号一致，方便直接复制去重跑）。
+		const missingIndexes = missingBaselineIndexes(meta, stat.seen ?? new Set());
 		return {
 			model,
 			category,
@@ -768,6 +807,8 @@ async function computeResults(dataDir) {
 			done,
 			configured,
 			notDone,
+			// baseline 专有：没做出来的题号（0 起，相对该题库）
+			missingIndexes,
 			// 本次运行的开始时间（ISO）；旧数据没有元数据时为 null，前端退回解析 displayName 里的运行戳
 			runStartedAt: meta !== undefined && typeof meta.startedAt === "string" ? meta.startedAt : null,
 			// 非 null 表示这一行跑的是 Baseline 题库子集，不是该任务全量
@@ -1093,6 +1134,8 @@ function apply(ctx) {
 				baselineTask: baselinePick !== null ? baselinePick.task : null,
 				baselinePicks: baselinePick !== null ? baselinePick.picks : null,
 				baselineSize: baselinePick !== null ? baselinePick.ids.length : null,
+				// 实际使用的有序 id 列表：/results 靠它算出"少做了哪几题"
+				baselineIds: baselinePick !== null ? baselinePick.ids : null,
 				startedAt: new Date().toISOString(),
 			}), "utf8");
 		} catch { /* 元数据写入失败不影响评测 */ }
@@ -1434,4 +1477,4 @@ function apply(ctx) {
 	}), `${name}: delete route`);
 }
 
-export { name, inject, apply, writeGeneratedModelConfig, readProviders, validateBenchName, clampedRangeLength, releaseQuestionCount, isEmptyAnswerLine };
+export { name, inject, apply, writeGeneratedModelConfig, readProviders, validateBenchName, clampedRangeLength, releaseQuestionCount, isEmptyAnswerLine, missingBaselineIndexes };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.2.21",
+  "version": "0.2.22",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place. Ships a Baseline probe set built from the questions a reference model cannot solve.",
   "license": "MIT",
   "author": "cszr (Vithrive)",