dsh-livebench-panel 0.1.2 → 0.1.4

package/lib/client.js

@@ -151,8 +151,21 @@ window.__ModuleLoader__.load({
 			const categories = useMemo(() => (config ? Object.keys(config.tasks) : []), [config]);
 			const taskList = useMemo(() => {
 				if (!config || sel.category.length === 0) return [];
-				return config.tasks[sel.category] ?? [];
+				return Object.keys(config.tasks[sel.category] ?? {});
 			}, [config, sel.category]);
+			// 有效题数：LiveBench 会丢弃「发布晚于所选 release」和「在所选 release
+			// 前已退役」的题目，这里按题目桶 (发布日, 移除日) 精确复算。
+			const countFor = useCallback((category, task) => {
+				if (!config || category.length === 0) return null;
+				const meta = (config.tasks[category] ?? {})[task];
+				if (!meta || !Array.isArray(meta.buckets)) return null;
+				const option = sel.release;
+				return meta.buckets.reduce((sum, b) => {
+					const releasedOk = b.r !== "" && b.r <= option;
+					const notRemoved = b.rm === "" || b.rm > option;
+					return sum + (releasedOk && notRemoved ? b.n : 0);
+				}, 0);
+			}, [config, sel.release]);
 			const selectedProvider = useMemo(
 				() => (config ? config.providers.find((p) => p.id === sel.provider) ?? null : null),
 				[config, sel.provider],
@@ -168,6 +181,10 @@ window.__ModuleLoader__.load({
 
 			const onStart = async () => {
 				setStartError(null);
+				if (sel.category !== "" && sel.task !== "" && countFor(sel.category, sel.task) === 0) {
+					setStartError(`任务 ${sel.task} 在 release ${sel.release} 下没有可用题目（该批题目已退役）。请换一个 release 或任务。`);
+					return;
+				}
 				setBusy(true);
 				try {
 					const payload = await api("/start", {
@@ -263,7 +280,11 @@ window.__ModuleLoader__.load({
 							h("label", { className: c("label") }, "任务"),
 							h("select", { className: c("select"), value: sel.task, onChange: setField("task") },
 								h("option", { value: "" }, sel.category === "" ? "（先选分类）" : "全部任务"),
-								taskList.map((task) => h("option", { key: task, value: task }, task))),
+								taskList.map((task) => {
+									const n = countFor(sel.category, task);
+									const label = n === null ? task : `${task}（${n} 题）`;
+									return h("option", { key: task, value: task, disabled: n === 0 }, n === 0 ? `${task}（此release无题）` : label);
+								})),
 						),
 						h("div", { className: c("field") },
 							h("label", { className: c("label") }, "题目序号范围（可选）"),

package/lib/index.js

@@ -227,18 +227,25 @@ function livebenchLayout() {
 function scanCategoryTasks(dataDir) {
 	const tasks = {};
 	if (!existsSync(dataDir)) return tasks;
-	let categoryDirs;
-	try {
-		categoryDirs = readDirSafe(dataDir);
-	} catch {
-		return tasks;
-	}
+	const categoryDirs = readDirSafe(dataDir);
 	for (const category of categoryDirs) {
 		const catDir = join(dataDir, category);
 		for (const task of readDirSafe(catDir)) {
-			if (existsSync(join(catDir, task, "question.jsonl"))) {
-				(tasks[category] ??= []).push(task);
+			const questionFile = join(catDir, task, "question.jsonl");
+			if (!existsSync(questionFile)) continue;
+			// Bucket questions by (release date, removal date) so the UI can
+			// compute how many questions survive a chosen release option:
+			// LiveBench drops questions released after the option and questions
+			// removed at or before the option.
+			const buckets = [];
+			for (const q of readJsonl(questionFile)) {
+				const r = typeof q.livebench_release_date === "string" ? q.livebench_release_date : "";
+				const rm = typeof q.livebench_removal_date === "string" ? q.livebench_removal_date : "";
+				const bucket = buckets.find((b) => b.r === r && b.rm === rm);
+				if (bucket) bucket.n += 1;
+				else buckets.push({ r, rm, n: 1 });
 			}
+			((tasks[category] ??= {})[task] = { buckets, total: buckets.reduce((sum, b) => sum + b.n, 0) });
 		}
 	}
 	return tasks;
@@ -403,7 +410,7 @@ function apply(ctx) {
 	/** The single run slot. */
 	let run = null;
 
-	const startRun = (body) => {
+	const startRun = async (body) => {
 		const layout = livebenchLayout();
 		if (!layout.available) {
 			return { status: 409, payload: { ok: false, error: `LiveBench venv not found under ${layout.root}` } };
@@ -473,11 +480,23 @@ function apply(ctx) {
 			PATH: `${join(layout.root, ".venv", "Scripts")}${delimiter}${process.env.PATH ?? ""}`,
 		};
 		// Only OpenAI-compatible providers can be routed with --api-base; for
-		// those, hand the key over via env (never the command line).
+		// those, hand the key over via env (never the command line). The key is
+		// resolved through the harness credential seam when available (values may
+		// live encrypted in .credentials.yaml rather than in the process env),
+		// falling back to the plain environment variable.
 		if (provider && provider.api === "openai-completions" && provider.baseURL) {
 			args.push("--api-base", provider.baseURL);
-			if (provider.keyEnv && process.env[provider.keyEnv]) {
-				env.LIVEBENCH_API_KEY = process.env[provider.keyEnv];
+			if (provider.keyEnv) {
+				let key;
+				try {
+					const credentials = ctx.get ? ctx.get("credentials") : undefined;
+					if (credentials && typeof credentials.resolve === "function") {
+						const resolved = await credentials.resolve(provider.keyEnv);
+						if (resolved && typeof resolved.value === "string" && resolved.value.length > 0) key = resolved.value;
+					}
+				} catch { /* credential seam unavailable — fall through to env */ }
+				if (!key && process.env[provider.keyEnv]) key = process.env[provider.keyEnv];
+				if (key) env.LIVEBENCH_API_KEY = key;
 			}
 		}
 
@@ -581,7 +600,7 @@ function apply(ctx) {
 				sendJson(res, 409, { ok: false, error: "another evaluation is already running" });
 				return;
 			}
-			const result = startRun(body);
+			const result = await startRun(body);
 			sendJson(res, result.status, result.payload);
 		},
 	}), `${name}: start route`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",