dsh-livebench-panel 0.1.9 → 0.1.11

package/lib/index.js

@@ -381,6 +381,14 @@ function writeGeneratedModelConfig(layout, YAML, { displayName, modelId, reasoni
 			// note: reasoning_effort is an OpenAI-style knob and is intentionally
 			// not forwarded on the anthropic protocol path.
 		}
+	} else if (protocol === "openai_responses") {
+		// OpenAI Responses API proxy: provider name matches
+		// get_api_function('openai_responses') → chat_completion_openai_responses,
+		// which converts reasoning_effort to reasoning:{effort} itself.
+		lines.push(`  openai_responses: ${modelId}`);
+		if (reasoningEffort) {
+			lines.push("api_kwargs:", "  default:", `    reasoning_effort: ${reasoningEffort}`);
+		}
 	} else {
 		lines.push(`  local: ${modelId}`);
 		if (reasoningEffort) {
@@ -494,6 +502,22 @@ function asInt(value, min, max, fallback) {
 	return Math.min(max, Math.max(min, Math.trunc(n)));
 }
 
+/**
+ * Validate one bench path: "live_bench", "live_bench/<category>" or
+ * "live_bench/<category>/<task>". Task names are mixed-case in LiveBench
+ * (AMPS_Hard, LCB_generation, …), so segments allow A-Za-z0-9_.
+ * @returns {string|null} error description, or null when valid.
+ */
+function validateBenchName(bn) {
+	const parts = bn.split("/").filter((s) => s.length > 0);
+	if (parts[0] !== "live_bench") return "must start with live_bench";
+	if (parts.length > 3) return "too deep";
+	for (const seg of parts) {
+		if (!/^[A-Za-z0-9_]{1,80}$/.test(seg)) return `bad segment "${seg}"`;
+	}
+	return null;
+}
+
 function apply(ctx) {
 	/** The single run slot. */
 	let run = null;
@@ -537,14 +561,9 @@ function apply(ctx) {
 			benchNames = [];
 			for (const bn of body.benchNames) {
 				if (typeof bn !== "string") return { status: 400, payload: { ok: false, error: "invalid bench name" } };
-				const parts = bn.split("/").filter((s) => s.length > 0);
-				if (parts[0] !== "live_bench" || parts.length > 3) {
-					return { status: 400, payload: { ok: false, error: `invalid bench path: ${bn}` } };
-				}
-				for (const seg of parts) {
-					if (!/^[a-z0-9_]+$/.test(seg)) return { status: 400, payload: { ok: false, error: `invalid bench path: ${bn}` } };
-				}
-				benchNames.push(parts.join("/"));
+				const error = validateBenchName(bn);
+				if (error) return { status: 400, payload: { ok: false, error: `invalid bench path: ${bn} (${error})` } };
+				benchNames.push(bn.split("/").filter((s) => s.length > 0).join("/"));
 			}
 		}
 
@@ -557,12 +576,19 @@ function apply(ctx) {
 		// selects LiveBench's native anthropic client and the spawn env points
 		// the SDK at the proxy (ANTHROPIC_BASE_URL / ANTHROPIC_API_KEY).
 		const isAnthropicRoute = provider && provider.api === "anthropic-messages" && provider.baseURL;
-		if (isAnthropicRoute || hasEffortSuffix) {
+		// OpenAI Responses-API proxies (api: openai-responses) select LiveBench's
+		// openai_responses client; --api-base + LIVEBENCH_API_KEY route them the
+		// usual way, and reasoning_effort converts to reasoning:{effort} inside.
+		const isOpenAIResponsesRoute = provider
+			&& (provider.api === "openai-responses" || provider.api === "openai_responses")
+			&& provider.baseURL;
+		const needConfig = isAnthropicRoute || isOpenAIResponsesRoute || hasEffortSuffix;
+		if (needConfig) {
 			writeError = writeGeneratedModelConfig(layout, loadYaml(profileDir), {
 				displayName,
 				modelId,
 				reasoningEffort: isAnthropicRoute ? null : reasoningEffort,
-				protocol: isAnthropicRoute ? "anthropic" : "openai",
+				protocol: isAnthropicRoute ? "anthropic" : isOpenAIResponsesRoute ? "openai_responses" : "openai",
 			});
 			if (writeError) return { status: 500, payload: { ok: false, error: writeError } };
 			cliModel = displayName;
@@ -693,7 +719,8 @@ function apply(ctx) {
 				providers: providers.map(({ id, name: pname, models, api, baseURL }) => ({
 					id,
 					name: pname,
-					routable: (api === "openai-completions" || api === "anthropic-messages") && typeof baseURL === "string" && baseURL.length > 0,
+					routable: ["openai-completions", "openai-responses", "openai_responses", "anthropic-messages"].includes(api)
+						&& typeof baseURL === "string" && baseURL.length > 0,
 					baseURL: baseURL ?? null,
 					models,
 				})),
@@ -870,7 +897,7 @@ function apply(ctx) {
 				const task = typeof row.task === "string" ? row.task : "";
 				// path-safety: membership in the scanned task list + charset guards
 				if (!/^[A-Za-z0-9._@-]{1,160}$/.test(model)) continue;
-				if (!/^[a-z0-9_]{1,60}$/.test(category) || !/^[a-z0-9_]{1,60}$/.test(task)) continue;
+				if (!/^[A-Za-z0-9_]{1,80}$/.test(category) || !/^[A-Za-z0-9_]{1,80}$/.test(task)) continue;
 				if (!(tasks[category] && tasks[category][task])) continue;
 				const taskDir = join(layout.dataDir, category, task);
 				removed += filterJsonlByModel(join(taskDir, "model_answer", `${model}.jsonl`), "model_id", model);
@@ -886,4 +913,4 @@ function runsAlive() {
 	return 1;
 }
 
-export { name, inject, apply, writeGeneratedModelConfig, readProviders };
+export { name, inject, apply, writeGeneratedModelConfig, readProviders, validateBenchName };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.9",
+  "version": "0.1.11",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",