dsh-livebench-panel 0.1.5 → 0.1.6

package/lib/client.js

@@ -45,7 +45,12 @@ window.__ModuleLoader__.load({
 .dlb_helpTable th,.dlb_helpTable td{text-align:left;vertical-align:top;padding:5px 8px;border-bottom:1px solid var(--dsw-alias-border-l2)}
 .dlb_helpTable th{color:var(--dsw-alias-label-tertiary);font-weight:500;white-space:nowrap}
 .dlb_helpTable td{color:var(--dsw-alias-label-secondary);line-height:1.55}
-.dlb_helpP{margin:0 0 8px;font-size:12px;color:var(--dsw-alias-label-secondary);line-height:1.6}`;
+.dlb_helpP{margin:0 0 8px;font-size:12px;color:var(--dsw-alias-label-secondary);line-height:1.6}
+.dlb_chips{display:flex;flex-wrap:wrap;gap:6px}
+.dlb_chip{appearance:none;font:inherit;cursor:pointer;color:var(--dsw-alias-label-secondary);background:transparent;border:1px solid var(--dsw-alias-border-l2);border-radius:999px;padding:3px 12px;font-size:12px;line-height:18px}
+.dlb_chip:disabled{opacity:.4;cursor:not-allowed}
+.dlb_chipOn{color:var(--dsw-alias-label-primary);border-color:var(--dsw-alias-state-business-primary);background:color-mix(in srgb,var(--dsw-alias-state-business-primary) 14%,transparent)}
+.dlb_drag{cursor:grab;color:var(--dsw-alias-label-tertiary);text-align:center;user-select:none}`;
 		const tagId = "dsh-livebench-panel/panel.css";
 		if (typeof document !== "undefined" && document.querySelector("style[data-plugin-css=" + JSON.stringify(tagId) + "]") === null) {
 			const tag = document.createElement("style");
@@ -87,13 +92,18 @@ window.__ModuleLoader__.load({
 		function LiveBenchView() {
 			const [config, setConfig] = useState(null);
 			const [configError, setConfigError] = useState(null);
-			const [sel, setSel] = useState({ provider: "", model: "", reasoning: "default", category: "", task: "", release: "2024-11-25", begin: "", end: "", maxTokens: "32000" });
+			const [sel, setSel] = useState({ provider: "", model: "", reasoning: "default", cats: [], tasks: [], release: "2024-11-25", begin: "", end: "", maxTokens: "32000" });
 			const [busy, setBusy] = useState(false);
 			const [running, setRunning] = useState(false);
 			const [log, setLog] = useState("");
 			const [exitCode, setExitCode] = useState(null);
 			const [startError, setStartError] = useState(null);
 			const [results, setResults] = useState(null);
+			const [pickedRows, setPickedRows] = useState(() => new Set());
+			const [order, setOrder] = useState(() => {
+				try { return JSON.parse(localStorage.getItem("dlb_result_order_v1")) ?? []; } catch { return []; }
+			});
+			const dragKey = useRef(null);
 			const logRef = useRef(null);
 
 			const loadConfig = useCallback(async () => {
@@ -158,10 +168,6 @@ window.__ModuleLoader__.load({
 			}, [log]);
 
 			const categories = useMemo(() => (config ? Object.keys(config.tasks) : []), [config]);
-			const taskList = useMemo(() => {
-				if (!config || sel.category.length === 0) return [];
-				return Object.keys(config.tasks[sel.category] ?? {});
-			}, [config, sel.category]);
 			// 有效题数：LiveBench 会丢弃「发布晚于所选 release」和「在所选 release
 			// 前已退役」的题目，这里按题目桶 (发布日, 移除日) 精确复算。
 			const countFor = useCallback((category, task) => {
@@ -175,6 +181,20 @@ window.__ModuleLoader__.load({
 					return sum + (releasedOk && notRemoved ? b.n : 0);
 				}, 0);
 			}, [config, sel.release]);
+			const catCount = useCallback((category) => {
+				if (!config) return 0;
+				return Object.keys(config.tasks[category] ?? {}).reduce((sum, task) => sum + (countFor(category, task) ?? 0), 0);
+			}, [config, countFor]);
+			// 多选任务 chips：未选分类时展示全部分类的任务
+			const taskChips = useMemo(() => {
+				const cats = sel.cats.length > 0 ? sel.cats : categories;
+				return cats.flatMap((cat) => Object.keys(config?.tasks[cat] ?? {}).map((task) => `${cat}/${task}`));
+			}, [sel.cats, categories, config]);
+			const toggleList = (key, value) => setSel((prev) => {
+				const list = prev[key];
+				const next = list.includes(value) ? list.filter((v) => v !== value) : [...list, value];
+				return { ...prev, [key]: next };
+			});
 			const selectedProvider = useMemo(
 				() => (config ? config.providers.find((p) => p.id === sel.provider) ?? null : null),
 				[config, sel.provider],
@@ -190,8 +210,17 @@ window.__ModuleLoader__.load({
 
 			const onStart = async () => {
 				setStartError(null);
-				if (sel.category !== "" && sel.task !== "" && countFor(sel.category, sel.task) === 0) {
-					setStartError(`任务 ${sel.task} 在 release ${sel.release} 下没有可用题目（该批题目已退役）。请换一个 release 或任务。`);
+				const benchNames = [];
+				if (sel.cats.length === 0) {
+					benchNames.push("live_bench");
+				} else if (sel.tasks.length === 0) {
+					for (const cat of sel.cats) benchNames.push(`live_bench/${cat}`);
+				} else {
+					for (const t of sel.tasks) benchNames.push(`live_bench/${t}`);
+				}
+				const zeroTask = sel.tasks.find((t) => countFor(t.split("/")[0], t.split("/")[1]) === 0);
+				if (zeroTask) {
+					setStartError(`任务 ${zeroTask.split("/")[1]} 在 release ${sel.release} 下没有可用题目（该批题目已退役），请取消勾选或换 release。`);
 					return;
 				}
 				setBusy(true);
@@ -203,8 +232,7 @@ window.__ModuleLoader__.load({
 							provider: sel.provider,
 							model: sel.model,
 							reasoningEffort: sel.reasoning,
-							category: sel.category,
-							task: sel.task,
+							benchNames,
 							release: sel.release,
 							begin: sel.begin,
 							end: sel.end,
@@ -233,22 +261,65 @@ window.__ModuleLoader__.load({
 						const provider = config?.providers.find((p) => p.id === value) ?? null;
 						next.model = provider?.models[0]?.id ?? "";
 					}
-					if (key === "category") next.task = "";
 					return next;
 				});
 			};
 
-			// results table: pivot rows -> model x task
-			const resultTable = useMemo(() => {
-				if (!results || results.rows.length === 0) return null;
-				const models = [...new Set(results.rows.map((r) => r.model))].sort();
-				const tasks = [...new Set(results.rows.map((r) => r.task))].sort();
-				const cell = (model, task) => {
-					const row = results.rows.find((r) => r.model === model && r.task === task);
-					return row ? `${row.score.toFixed(1)} (${row.judged}/${row.total})` : "—";
-				};
-				return { models, tasks, cell };
-			}, [results]);
+			// ---- 成绩行：排序（拖拽，localStorage 持久化）/ 选择 / 删除 ----
+			const rowKey = (row) => `${row.model}|${row.category}|${row.task}`;
+			const sortedRows = useMemo(() => {
+				if (!results) return [];
+				const indexOf = new Map(order.map((key, index) => [key, index]));
+				return [...results.rows].sort((a, b) => {
+					const ia = indexOf.has(rowKey(a)) ? indexOf.get(rowKey(a)) : Number.MAX_SAFE_INTEGER;
+					const ib = indexOf.has(rowKey(b)) ? indexOf.get(rowKey(b)) : Number.MAX_SAFE_INTEGER;
+					if (ia !== ib) return ia - ib;
+					return (b.time ?? 0) - (a.time ?? 0);
+				});
+			}, [results, order]);
+
+			const persistOrder = (keys) => {
+				setOrder(keys);
+				try { localStorage.setItem("dlb_result_order_v1", JSON.stringify(keys)); } catch { /* private mode */ }
+			};
+
+			const onRowDrop = (targetKey) => {
+				const sourceKey = dragKey.current;
+				dragKey.current = null;
+				if (!sourceKey || sourceKey === targetKey) return;
+				const keys = sortedRows.map(rowKey);
+				const from = keys.indexOf(sourceKey);
+				const to = keys.indexOf(targetKey);
+				if (from < 0 || to < 0) return;
+				keys.splice(to, 0, keys.splice(from, 1)[0]);
+				persistOrder(keys);
+			};
+
+			const togglePicked = (key) => setPickedRows((prev) => {
+				const next = new Set(prev);
+				if (next.has(key)) next.delete(key); else next.add(key);
+				return next;
+			});
+
+			const deleteRows = async (rows) => {
+				if (rows.length === 0) return;
+				if (!window.confirm(`确认删除 ${rows.length} 条评测成绩？将同时删除对应的模型答案记录，不可恢复。`)) return;
+				await api("/delete", {
+					method: "POST",
+					headers: { "content-type": "application/json" },
+					body: JSON.stringify({ rows: rows.map((r) => ({ model: r.model, category: r.category, task: r.task })) }),
+				});
+				setPickedRows(new Set());
+				persistOrder([...order].filter((key) => !rows.some((r) => rowKey(r) === key)));
+				await loadResults();
+			};
+
+			const fmtTime = (t) => {
+				if (!t) return "—";
+				const dt = new Date(t * 1000);
+				const pad = (n) => String(n).padStart(2, "0");
+				return `${dt.getFullYear()}-${pad(dt.getMonth() + 1)}-${pad(dt.getDate())} ${pad(dt.getHours())}:${pad(dt.getMinutes())}`;
+			};
 
 			return h("div", { className: c("root") },
 				h("div", { className: c("head") },
@@ -279,20 +350,30 @@ window.__ModuleLoader__.load({
 							h("select", { className: c("select"), value: sel.release, onChange: setField("release") },
 								(config?.releases ?? ["2024-11-25"]).map((r) => h("option", { key: r, value: r }, r))),
 						),
-						h("div", { className: c("field") },
-							h("label", { className: c("label") }, "分类"),
-							h("select", { className: c("select"), value: sel.category, onChange: setField("category") },
-								h("option", { value: "" }, "全部分类"),
-								categories.map((cat) => h("option", { key: cat, value: cat }, cat))),
+						h("div", { className: c("field"), style: { gridColumn: "1 / -1" } },
+							h("label", { className: c("label") }, "分类（可多选，不选 = 全部分类；括号内为该 release 下有效题数）"),
+							h("div", { className: c("chips") },
+								categories.map((cat) => {
+									const n = catCount(cat);
+									const active = sel.cats.includes(cat);
+									return h("button", {
+										key: cat, type: "button", className: c("chip") + (active ? " " + c("chipOn") : ""),
+										onClick: () => toggleList("cats", cat), disabled: n === 0,
+									}, `${cat}（${n}）`);
+								})),
 						),
-						h("div", { className: c("field") },
-							h("label", { className: c("label") }, "任务"),
-							h("select", { className: c("select"), value: sel.task, onChange: setField("task") },
-								h("option", { value: "" }, sel.category === "" ? "（先选分类）" : "全部任务"),
-								taskList.map((task) => {
-									const n = countFor(sel.category, task);
-									const label = n === null ? task : `${task}（${n} 题）`;
-									return h("option", { key: task, value: task, disabled: n === 0 }, n === 0 ? `${task}（此release无题）` : label);
+						sel.cats.length > 0 && h("div", { className: c("field"), style: { gridColumn: "1 / -1" } },
+							h("label", { className: c("label") }, "任务（可多选，不选 = 所选分类的全部任务）"),
+							h("div", { className: c("chips") },
+								taskChips.map((key) => {
+									const [cat, task] = key.split("/");
+									const n = countFor(cat, task) ?? 0;
+									const active = sel.tasks.includes(key);
+									return h("button", {
+										key, type: "button", className: c("chip") + (active ? " " + c("chipOn") : ""),
+										onClick: () => toggleList("tasks", key), disabled: n === 0,
+										title: n === 0 ? "该任务在此 release 下已无可用题目" : `${n} 题有效`,
+									}, `${task}（${n}）`);
 								})),
 						),
 						h("div", { className: c("field") },
@@ -324,18 +405,52 @@ window.__ModuleLoader__.load({
 					),
 					h("pre", { className: c("log"), ref: logRef }, log || "（暂无输出）"),
 				),
-				resultTable !== null && h("div", { className: c("card") },
+				sortedRows.length > 0 && h("div", { className: c("card") },
 					h("div", { className: c("row") },
-						h("span", { className: c("label") }, "评测成绩（分数 = 平均分 ×100，括号内为 已判/总题数）"),
+						h("span", { className: c("label") }, "评测成绩（分数 = 平均分 ×100；行首可拖动排序，顺序保存在本机）"),
+						h("button", {
+							className: c("btn") + " " + c("btnDanger"),
+							disabled: pickedRows.size === 0,
+							onClick: () => deleteRows(sortedRows.filter((r) => pickedRows.has(rowKey(r)))),
+						}, `删除所选（${pickedRows.size}）`),
 					),
 					h("table", { className: c("table") },
 						h("thead", null, h("tr", null,
+							h("th", { style: { width: "30px" } }, "⠿"),
+							h("th", { style: { width: "30px" } },
+								h("input", { type: "checkbox", checked: pickedRows.size > 0 && pickedRows.size === sortedRows.length,
+									onChange: (event) => setPickedRows(event.target.checked ? new Set(sortedRows.map(rowKey)) : new Set()) })),
+							h("th", null, "time"),
 							h("th", null, "model"),
-							resultTable.tasks.map((task) => h("th", { key: task }, task)))),
+							h("th", null, "category"),
+							h("th", null, "task"),
+							h("th", null, "score"),
+							h("th", null, "操作"),
+						)),
 						h("tbody", null,
-							resultTable.models.map((model) => h("tr", { key: model },
-								h("td", null, model),
-								resultTable.tasks.map((task) => h("td", { key: task, className: c("score") }, resultTable.cell(model, task)))))),
+							sortedRows.map((row) => {
+								const key = rowKey(row);
+								const checked = pickedRows.has(key);
+								return h("tr", {
+									key,
+									draggable: true,
+									onDragStart: () => { dragKey.current = key; },
+									onDragOver: (event) => event.preventDefault(),
+									onDrop: () => onRowDrop(key),
+								},
+									h("td", { className: c("drag"), title: "拖动排序" }, "⠿"),
+									h("td", null, h("input", { type: "checkbox", checked, onChange: () => togglePicked(key) })),
+									h("td", null, fmtTime(row.time)),
+									h("td", null, row.model),
+									h("td", null, row.category),
+									h("td", null, row.task),
+									h("td", { className: c("score") }, `${row.score.toFixed(1)} (${row.judged}/${row.total})`),
+									h("td", null, h("button", {
+										className: c("btnGhost") + " " + c("btn"), title: "删除此条成绩",
+										onClick: () => deleteRows([row]),
+									}, "删除")),
+								);
+							})),
 					),
 				),
 				config !== null && config.available === false && h("div", { className: c("card") + " " + c("notice") },

package/lib/index.js

@@ -365,7 +365,7 @@ function readJsonl(path) {
  * files, plus the judged/total question counts per task.
  */
 function computeResults(dataDir) {
-	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n}
+	const judged = new Map(); // `${model}\u0000${task}` -> {model, category, task, sum, n, time}
 	const totals = new Map(); // task -> question count
 	if (!existsSync(dataDir)) return { rows: [], taskTotals: {} };
 	for (const category of readDirSafe(dataDir)) {
@@ -380,9 +380,11 @@ function computeResults(dataDir) {
 				const score = typeof row.score === "number" ? row.score : Number(row.score);
 				if (model === null || !Number.isFinite(score) || score < 0) continue;
 				const key = `${model}\u0000${task}`;
-				const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0 };
+				const entry = judged.get(key) ?? { model, category, task, sum: 0, n: 0, time: 0 };
 				entry.sum += score;
 				entry.n += 1;
+				const tstamp = Number(row.tstamp);
+				if (Number.isFinite(tstamp) && tstamp > entry.time) entry.time = tstamp;
 				judged.set(key, entry);
 			}
 		}
@@ -394,11 +396,38 @@ function computeResults(dataDir) {
 		score: entry.n > 0 ? (entry.sum / entry.n) * 100 : 0,
 		judged: entry.n,
 		total: totals.get(entry.task) ?? 0,
+		time: entry.time > 0 ? entry.time : null,
 	}));
 	const taskTotals = Object.fromEntries(totals);
 	return { rows, taskTotals };
 }
 
+/**
+ * Remove every JSONL line of one model from a file (matched on the given
+ * field), rewriting the file only when something changed.
+ * @returns {number} removed line count.
+ */
+function filterJsonlByModel(path, field, model) {
+	if (!existsSync(path)) return 0;
+	const lines = readFileSync(path, "utf8").split(/\r?\n/);
+	const kept = [];
+	let removed = 0;
+	for (const line of lines) {
+		if (line.trim().length === 0) continue;
+		let match = false;
+		try {
+			const parsed = JSON.parse(line);
+			match = parsed[field] === model;
+		} catch {
+			match = false;
+		}
+		if (match) removed += 1;
+		else kept.push(line);
+	}
+	if (removed > 0) writeFileSync(path, kept.length > 0 ? kept.join("\n") + "\n" : "", "utf8");
+	return removed;
+}
+
 /** Clamp helper for query params. */
 function asInt(value, min, max, fallback) {
 	const n = Number(value);
@@ -438,10 +467,28 @@ function apply(ctx) {
 		const providers = readProviders(profileDir);
 		const provider = providers.find((p) => p.id === providerId) ?? null;
 
-		// A selected reasoning effort is passed through LiveBench's model-config
-		// api_kwargs (chat_completion_openai merges them into the SDK call). The
-		// effort is encoded in the display name so different efforts land in
-		// separate result rows.
+		// Multi-select support: the client resolves its category/task chips into
+		// an explicit bench-path list ("live_bench", "live_bench/<cat>",
+		// "live_bench/<cat>/<task>"); gen_api_answer accepts nargs="+".
+		let benchNames = null;
+		if (Array.isArray(body.benchNames) && body.benchNames.length > 0) {
+			if (body.benchNames.length > 24) {
+				return { status: 400, payload: { ok: false, error: "too many bench names (max 24)" } };
+			}
+			benchNames = [];
+			for (const bn of body.benchNames) {
+				if (typeof bn !== "string") return { status: 400, payload: { ok: false, error: "invalid bench name" } };
+				const parts = bn.split("/").filter((s) => s.length > 0);
+				if (parts[0] !== "live_bench" || parts.length > 3) {
+					return { status: 400, payload: { ok: false, error: `invalid bench path: ${bn}` } };
+				}
+				for (const seg of parts) {
+					if (!/^[a-z0-9_]+$/.test(seg)) return { status: 400, payload: { ok: false, error: `invalid bench path: ${bn}` } };
+				}
+				benchNames.push(parts.join("/"));
+			}
+		}
+
 		const hasEffortSuffix = reasoningEffort !== null && reasoningEffort !== "off";
 		const displayName = displayModelName(providerId || "direct", modelId) + (hasEffortSuffix ? "@" + reasoningEffort : "");
 		let cliModel = modelId;
@@ -456,7 +503,8 @@ function apply(ctx) {
 			"run_livebench.py",
 			"--model", cliModel,
 			"--model-display-name", displayName,
-			"--bench-name", benchParts.join("/"),
+			"--bench-name",
+			...(benchNames !== null ? benchNames : [benchParts.join("/")]),
 			"--livebench-release-option", release,
 			"--max-tokens", String(asInt(body.maxTokens, 256, 32768, 32000)),
 			"--parallel-requests", String(asInt(body.parallel, 1, 8, 2)),
@@ -669,6 +717,46 @@ function apply(ctx) {
 			sendJson(res, 200, { ok: true, rows, taskTotals });
 		},
 	}), `${name}: results route`);
+
+	ctx.effect(() => ctx.webServer.register({
+		kind: "exact",
+		path: `${API}/delete`,
+		handler: async (req, res) => {
+			if (req.method !== "POST") {
+				sendJson(res, 405, { ok: false, error: "method not allowed" });
+				return;
+			}
+			if (!originAllowed(req)) {
+				sendJson(res, 403, { ok: false, error: "forbidden origin" });
+				return;
+			}
+			let body;
+			try {
+				body = JSON.parse(await readBody(req));
+			} catch {
+				sendJson(res, 400, { ok: false, error: "invalid request body" });
+				return;
+			}
+			const rows = Array.isArray(body.rows) ? body.rows.slice(0, 200) : [];
+			const layout = livebenchLayout();
+			const tasks = scanCategoryTasks(layout.dataDir);
+			let removed = 0;
+			for (const row of rows) {
+				if (!row || typeof row !== "object") continue;
+				const model = typeof row.model === "string" ? row.model : "";
+				const category = typeof row.category === "string" ? row.category : "";
+				const task = typeof row.task === "string" ? row.task : "";
+				// path-safety: membership in the scanned task list + charset guards
+				if (!/^[A-Za-z0-9._@-]{1,160}$/.test(model)) continue;
+				if (!/^[a-z0-9_]{1,60}$/.test(category) || !/^[a-z0-9_]{1,60}$/.test(task)) continue;
+				if (!(tasks[category] && tasks[category][task])) continue;
+				const taskDir = join(layout.dataDir, category, task);
+				removed += filterJsonlByModel(join(taskDir, "model_answer", `${model}.jsonl`), "model_id", model);
+				removed += filterJsonlByModel(join(taskDir, "model_judgment", "ground_truth_judgment.jsonl"), "model", model);
+			}
+			sendJson(res, 200, { ok: true, removed });
+		},
+	}), `${name}: delete route`);
 }
 
 /** Count alive runs (at most one today, kept for future parallel lanes). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",