dsh-livebench-panel 0.1.3 → 0.1.4

package/lib/client.js

@@ -151,8 +151,21 @@ window.__ModuleLoader__.load({
 			const categories = useMemo(() => (config ? Object.keys(config.tasks) : []), [config]);
 			const taskList = useMemo(() => {
 				if (!config || sel.category.length === 0) return [];
-				return config.tasks[sel.category] ?? [];
+				return Object.keys(config.tasks[sel.category] ?? {});
 			}, [config, sel.category]);
+			// 有效题数：LiveBench 会丢弃「发布晚于所选 release」和「在所选 release
+			// 前已退役」的题目，这里按题目桶 (发布日, 移除日) 精确复算。
+			const countFor = useCallback((category, task) => {
+				if (!config || category.length === 0) return null;
+				const meta = (config.tasks[category] ?? {})[task];
+				if (!meta || !Array.isArray(meta.buckets)) return null;
+				const option = sel.release;
+				return meta.buckets.reduce((sum, b) => {
+					const releasedOk = b.r !== "" && b.r <= option;
+					const notRemoved = b.rm === "" || b.rm > option;
+					return sum + (releasedOk && notRemoved ? b.n : 0);
+				}, 0);
+			}, [config, sel.release]);
 			const selectedProvider = useMemo(
 				() => (config ? config.providers.find((p) => p.id === sel.provider) ?? null : null),
 				[config, sel.provider],
@@ -168,6 +181,10 @@ window.__ModuleLoader__.load({
 
 			const onStart = async () => {
 				setStartError(null);
+				if (sel.category !== "" && sel.task !== "" && countFor(sel.category, sel.task) === 0) {
+					setStartError(`任务 ${sel.task} 在 release ${sel.release} 下没有可用题目（该批题目已退役）。请换一个 release 或任务。`);
+					return;
+				}
 				setBusy(true);
 				try {
 					const payload = await api("/start", {
@@ -263,7 +280,11 @@ window.__ModuleLoader__.load({
 							h("label", { className: c("label") }, "任务"),
 							h("select", { className: c("select"), value: sel.task, onChange: setField("task") },
 								h("option", { value: "" }, sel.category === "" ? "（先选分类）" : "全部任务"),
-								taskList.map((task) => h("option", { key: task, value: task }, task))),
+								taskList.map((task) => {
+									const n = countFor(sel.category, task);
+									const label = n === null ? task : `${task}（${n} 题）`;
+									return h("option", { key: task, value: task, disabled: n === 0 }, n === 0 ? `${task}（此release无题）` : label);
+								})),
 						),
 						h("div", { className: c("field") },
 							h("label", { className: c("label") }, "题目序号范围（可选）"),

package/lib/index.js

@@ -227,18 +227,25 @@ function livebenchLayout() {
 function scanCategoryTasks(dataDir) {
 	const tasks = {};
 	if (!existsSync(dataDir)) return tasks;
-	let categoryDirs;
-	try {
-		categoryDirs = readDirSafe(dataDir);
-	} catch {
-		return tasks;
-	}
+	const categoryDirs = readDirSafe(dataDir);
 	for (const category of categoryDirs) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
-			if (existsSync(join(catDir, task, "question.jsonl"))) {
-				(tasks[category] ??= []).push(task);
+			const questionFile = join(catDir, task, "question.jsonl");
+			if (!existsSync(questionFile)) continue;
+			// Bucket questions by (release date, removal date) so the UI can
+			// compute how many questions survive a chosen release option:
+			// LiveBench drops questions released after the option and questions
+			// removed at or before the option.
+			const buckets = [];
+			for (const q of readJsonl(questionFile)) {
+				const r = typeof q.livebench_release_date === "string" ? q.livebench_release_date : "";
+				const rm = typeof q.livebench_removal_date === "string" ? q.livebench_removal_date : "";
+				const bucket = buckets.find((b) => b.r === r && b.rm === rm);
+				if (bucket) bucket.n += 1;
+				else buckets.push({ r, rm, n: 1 });
 			}
+			((tasks[category] ??= {})[task] = { buckets, total: buckets.reduce((sum, b) => sum + b.n, 0) });
 		}
 	}
 	return tasks;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",