dsh-livebench-panel 0.1.8 → 0.1.10

package/lib/client.js

@@ -448,7 +448,7 @@ window.__ModuleLoader__.load({
 						running && h("button", { className: c("btn") + " " + c("btnDanger"), onClick: onStop }, "停止"),
 						h("button", { className: c("btnGhost") + " " + c("btn"), onClick: () => { loadConfig(); loadResults(); } }, "刷新"),
 						selectedProvider && !selectedProvider.routable && h("span", { className: c("hint") },
-							"该 provider 未配置 openai-completions baseURL，LiveBench 将按模型名原生路由（未注册的模型名会失败）。"),
+							"该 provider 的协议或端点未知，无法自动路由：LiveBench 将按模型名原生尝试，未注册的模型名会失败。"),
 					),
 					startError !== null && h("p", { className: c("error") }, startError),
 				),

package/lib/index.js

@@ -176,10 +176,10 @@ function readProviders(profileDir) {
 	if (piAiData !== null) {
 		for (const provider of providers) {
 			if (provider.baseURL === null) {
-				const base = builtinBaseUrl(piAiData, provider.id);
-				if (base !== null) {
-					provider.baseURL = base;
-					provider.api = provider.api ?? "openai-completions";
+				const info = builtinProtocolInfo(piAiData, provider.id);
+				if (info) {
+					provider.baseURL = info.baseURL;
+					provider.api = info.api;
 				}
 			}
 		}
@@ -215,17 +215,25 @@ function resolvePiAiDataDir(profileDir) {
 	return null;
 }
 
-/** Read one provider's built-in OpenAI-compatible baseUrl from pi-ai data. */
-function builtinBaseUrl(piAiDataDir, providerId) {
+/**
+ * Read one provider's built-in protocol + baseUrl from pi-ai data.
+ * Prefers OpenAI Chat Completions; Anthropic Messages is also routable
+ * (LiveBench talks it natively and honors ANTHROPIC_BASE_URL/ANTHROPIC_API_KEY).
+ */
+function builtinProtocolInfo(piAiDataDir, providerId) {
 	if (!/^[a-z0-9-]+$/.test(providerId)) return null;
 	const file = join(piAiDataDir, `${providerId}.json`);
 	if (!existsSync(file)) return null;
 	try {
 		const data = JSON.parse(readFileSync(file, "utf8"));
-		const openai = data?.["openai-completions"];
-		if (openai && typeof openai === "object") {
-			for (const entry of Object.values(openai)) {
-				if (entry && typeof entry.baseUrl === "string" && entry.baseUrl.length > 0) return entry.baseUrl;
+		for (const api of ["openai-completions", "anthropic-messages"]) {
+			const group = data?.[api];
+			if (group && typeof group === "object") {
+				for (const entry of Object.values(group)) {
+					if (entry && typeof entry.baseUrl === "string" && entry.baseUrl.length > 0) {
+						return { api, baseURL: entry.baseUrl };
+					}
+				}
 			}
 		}
 	} catch { /* unreadable data file — treat as unknown */ }
@@ -355,21 +363,40 @@ function displayModelName(providerId, modelId) {
  * The file is regenerated on every start; secrets never go in here.
  * @returns {string|null} error message, or null on success.
  */
-function writeGeneratedModelConfig(layout, YAML, { displayName, modelId, reasoningEffort }) {
+function writeGeneratedModelConfig(layout, YAML, { displayName, modelId, reasoningEffort, protocol }) {
 	const configDir = join(layout.livebenchDir, "model", "model_configs");
 	const target = join(configDir, "dsh_panel_generated.yaml");
 	void YAML;
-	const doc = [
+	const lines = [
 		"# Generated by dsh-livebench-panel — regenerated on every evaluation start.",
 		"---",
 		`display_name: ${displayName}`,
 		"api_name:",
-		"  local: " + modelId,
-		"api_kwargs:",
-		"  default:",
-		`    reasoning_effort: ${reasoningEffort}`,
-		"",
-	].join("\n");
+	];
+	if (protocol === "anthropic") {
+		// Route through LiveBench's native anthropic client; the endpoint is
+		// pointed at the provider proxy via ANTHROPIC_BASE_URL (spawn env).
+		lines.push(`  anthropic: ${modelId}`, "default_provider: anthropic");
+		if (reasoningEffort) {
+			// note: reasoning_effort is an OpenAI-style knob and is intentionally
+			// not forwarded on the anthropic protocol path.
+		}
+	} else if (protocol === "openai_responses") {
+		// OpenAI Responses API proxy: provider name matches
+		// get_api_function('openai_responses') → chat_completion_openai_responses,
+		// which converts reasoning_effort to reasoning:{effort} itself.
+		lines.push(`  openai_responses: ${modelId}`);
+		if (reasoningEffort) {
+			lines.push("api_kwargs:", "  default:", `    reasoning_effort: ${reasoningEffort}`);
+		}
+	} else {
+		lines.push(`  local: ${modelId}`);
+		if (reasoningEffort) {
+			lines.push("api_kwargs:", "  default:", `    reasoning_effort: ${reasoningEffort}`);
+		}
+	}
+	lines.push("");
+	const doc = lines.join("\n");
 	try {
 		if (!existsSync(configDir)) return `model_configs directory not found: ${configDir}`;
 		writeFileSync(target, doc, "utf8");
@@ -532,8 +559,26 @@ function apply(ctx) {
 		const hasEffortSuffix = reasoningEffort !== null && reasoningEffort !== "off";
 		const displayName = displayModelName(providerId || "direct", modelId) + (hasEffortSuffix ? "@" + reasoningEffort : "");
 		let cliModel = modelId;
-		if (hasEffortSuffix) {
-			const writeError = writeGeneratedModelConfig(layout, loadYaml(profileDir), { displayName, modelId, reasoningEffort });
+		let writeError = null;
+		// Anthropic-protocol proxies cannot go through --api-base (that path
+		// speaks OpenAI Chat Completions). Instead the generated model config
+		// selects LiveBench's native anthropic client and the spawn env points
+		// the SDK at the proxy (ANTHROPIC_BASE_URL / ANTHROPIC_API_KEY).
+		const isAnthropicRoute = provider && provider.api === "anthropic-messages" && provider.baseURL;
+		// OpenAI Responses-API proxies (api: openai-responses) select LiveBench's
+		// openai_responses client; --api-base + LIVEBENCH_API_KEY route them the
+		// usual way, and reasoning_effort converts to reasoning:{effort} inside.
+		const isOpenAIResponsesRoute = provider
+			&& (provider.api === "openai-responses" || provider.api === "openai_responses")
+			&& provider.baseURL;
+		const needConfig = isAnthropicRoute || isOpenAIResponsesRoute || hasEffortSuffix;
+		if (needConfig) {
+			writeError = writeGeneratedModelConfig(layout, loadYaml(profileDir), {
+				displayName,
+				modelId,
+				reasoningEffort: isAnthropicRoute ? null : reasoningEffort,
+				protocol: isAnthropicRoute ? "anthropic" : isOpenAIResponsesRoute ? "openai_responses" : "openai",
+			});
 			if (writeError) return { status: 500, payload: { ok: false, error: writeError } };
 			cliModel = displayName;
 		}
@@ -567,15 +612,13 @@ function apply(ctx) {
 			// livebench import (shortuuid etc.).
 			PATH: `${join(layout.root, ".venv", "Scripts")}${delimiter}${process.env.PATH ?? ""}`,
 		};
-		// Only OpenAI-compatible providers can be routed with --api-base; for
-		// those, hand the key over via env (never the command line). The key is
-		// resolved through the harness credential seam when available (values may
-		// live encrypted in .credentials.yaml rather than in the process env),
-		// falling back to the plain environment variable.
-		if (provider && provider.api === "openai-completions" && provider.baseURL) {
-			args.push("--api-base", provider.baseURL);
+		// Provider routing: keys are resolved through the harness credential seam
+		// when available (values may live encrypted in .credentials.yaml rather
+		// than in the process env), falling back to the plain environment
+		// variable. Secrets travel via env only — never the command line.
+		if (provider && provider.baseURL) {
+			let key;
 			if (provider.keyEnv) {
-				let key;
 				try {
 					const credentials = ctx.get ? ctx.get("credentials") : undefined;
 					if (credentials && typeof credentials.resolve === "function") {
@@ -584,7 +627,15 @@ function apply(ctx) {
 					}
 				} catch { /* credential seam unavailable — fall through to env */ }
 				if (!key && process.env[provider.keyEnv]) key = process.env[provider.keyEnv];
+			}
+			if (provider.api === "openai-completions") {
+				args.push("--api-base", provider.baseURL);
 				if (key) env.LIVEBENCH_API_KEY = key;
+			} else if (provider.api === "anthropic-messages") {
+				// The generated model config selects the native anthropic client;
+				// the Anthropic SDK picks endpoint+key up from these env vars.
+				if (key) env.ANTHROPIC_API_KEY = key;
+				env.ANTHROPIC_BASE_URL = provider.baseURL;
 			}
 		}
 
@@ -657,7 +708,8 @@ function apply(ctx) {
 				providers: providers.map(({ id, name: pname, models, api, baseURL }) => ({
 					id,
 					name: pname,
-					routable: api === "openai-completions" && typeof baseURL === "string" && baseURL.length > 0,
+					routable: ["openai-completions", "openai-responses", "openai_responses", "anthropic-messages"].includes(api)
+						&& typeof baseURL === "string" && baseURL.length > 0,
 					baseURL: baseURL ?? null,
 					models,
 				})),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",