dsh-livebench-panel 0.1.1 → 0.1.3

package/lib/client.js

@@ -78,7 +78,7 @@ window.__ModuleLoader__.load({
 		function LiveBenchView() {
 			const [config, setConfig] = useState(null);
 			const [configError, setConfigError] = useState(null);
-			const [sel, setSel] = useState({ provider: "", model: "", category: "", task: "", release: "2024-11-25", begin: "", end: "", maxTokens: "4096" });
+			const [sel, setSel] = useState({ provider: "", model: "", reasoning: "default", category: "", task: "", release: "2024-11-25", begin: "", end: "", maxTokens: "32000" });
 			const [busy, setBusy] = useState(false);
 			const [running, setRunning] = useState(false);
 			const [log, setLog] = useState("");
@@ -273,7 +273,7 @@ window.__ModuleLoader__.load({
 							),
 						),
 						h("div", { className: c("field") },
-							h("label", { className: c("label") }, "max-tokens"),
+							h("label", { className: c("label") }, "max-tokens（默认 32000）"),
 							h("input", { className: c("input"), type: "number", min: 256, max: 32768, value: sel.maxTokens, onChange: setField("maxTokens") }),
 						),
 					),

package/lib/index.js

@@ -27,8 +27,8 @@
 import { spawn } from "node:child_process";
 import { readdirSync, readFileSync, existsSync, statSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
+import { delimiter, dirname, join } from "node:path";
 import { homedir } from "node:os";
-import { join } from "node:path";
 import { fileURLToPath } from "node:url";
 
 /** Stable Cordis plugin name. */
@@ -115,9 +115,70 @@ function readProviders(profileDir) {
 		keyEnv: typeof dsCfg.apiKeyEnv === "string" && dsCfg.apiKeyEnv.length > 0 ? dsCfg.apiKeyEnv : "DEEPSEEK_API_KEY",
 		models: dsModels,
 	});
+
+	// Providers without an explicit baseURL may still be OpenAI-compatible with
+	// a well-known endpoint: resolve it from the pi-ai provider registry
+	// (@earendil-works/pi-ai, the same source the harness serves from).
+	const piAiData = resolvePiAiDataDir(profileDir);
+	if (piAiData !== null) {
+		for (const provider of providers) {
+			if (provider.baseURL === null) {
+				const base = builtinBaseUrl(piAiData, provider.id);
+				if (base !== null) {
+					provider.baseURL = base;
+					provider.api = provider.api ?? "openai-completions";
+				}
+			}
+		}
+	}
 	return providers;
 }
 
+/**
+ * Locate the pi-ai provider data directory (providers/data/*.json).
+ * @returns {string|null} directory path, or null when pi-ai cannot be found.
+ */
+function resolvePiAiDataDir(profileDir) {
+	const candidates = [];
+	// 1) resolvable from the profile's own dependency tree
+	try {
+		const requireFromProfile = createRequire(join(profileDir, "package.json"));
+		candidates.push(join(dirname(requireFromProfile.resolve("@earendil-works/pi-ai/package.json")), "dist", "providers", "data"));
+	} catch { /* not in the profile tree */ }
+	// 2) the dsh installation the web process booted from (process.argv[1] = .../dsh/lib/bin.js)
+	const bin = process.argv[1];
+	if (typeof bin === "string" && bin.includes("@deepseek-ai")) {
+		const dshRoot = dirname(dirname(bin));
+		candidates.push(join(dshRoot, "node_modules", "@earendil-works", "pi-ai", "dist", "providers", "data"));
+	}
+	// 3) the default global npm layout on Windows
+	const appdata = process.env.APPDATA;
+	if (appdata) {
+		candidates.push(join(appdata, "npm", "node_modules", "@deepseek-ai", "dsh", "node_modules", "@earendil-works", "pi-ai", "dist", "providers", "data"));
+	}
+	for (const candidate of candidates) {
+		if (existsSync(candidate)) return candidate;
+	}
+	return null;
+}
+
+/** Read one provider's built-in OpenAI-compatible baseUrl from pi-ai data. */
+function builtinBaseUrl(piAiDataDir, providerId) {
+	if (!/^[a-z0-9-]+$/.test(providerId)) return null;
+	const file = join(piAiDataDir, `${providerId}.json`);
+	if (!existsSync(file)) return null;
+	try {
+		const data = JSON.parse(readFileSync(file, "utf8"));
+		const openai = data?.["openai-completions"];
+		if (openai && typeof openai === "object") {
+			for (const entry of Object.values(openai)) {
+				if (entry && typeof entry.baseUrl === "string" && entry.baseUrl.length > 0) return entry.baseUrl;
+			}
+		}
+	} catch { /* unreadable data file — treat as unknown */ }
+	return null;
+}
+
 /** Reasoning efforts offered by the built-in DeepSeek route. */
 const DEEPSEEK_EFFORTS = ["off", "low", "high", "max"];
 /** Fallback catalog when settings.yaml has no llm-deepseek.models section. */
@@ -342,7 +403,7 @@ function apply(ctx) {
 	/** The single run slot. */
 	let run = null;
 
-	const startRun = (body) => {
+	const startRun = async (body) => {
 		const layout = livebenchLayout();
 		if (!layout.available) {
 			return { status: 409, payload: { ok: false, error: `LiveBench venv not found under ${layout.root}` } };
@@ -390,7 +451,7 @@ function apply(ctx) {
 			"--model-display-name", displayName,
 			"--bench-name", benchParts.join("/"),
 			"--livebench-release-option", release,
-			"--max-tokens", String(asInt(body.maxTokens, 256, 32768, 4096)),
+			"--max-tokens", String(asInt(body.maxTokens, 256, 32768, 32000)),
 			"--parallel-requests", String(asInt(body.parallel, 1, 8, 2)),
 			"--mode", "single",
 		];
@@ -406,13 +467,29 @@ function apply(ctx) {
 			...process.env,
 			HF_HOME: join(layout.root, ".hf_cache"),
 			HF_HUB_DISABLE_SYMLINKS_WARNING: "1",
+			// run_livebench.py shells out to bare `python`; without the venv on
+			// PATH that resolves to the system interpreter and dies on the first
+			// livebench import (shortuuid etc.).
+			PATH: `${join(layout.root, ".venv", "Scripts")}${delimiter}${process.env.PATH ?? ""}`,
 		};
 		// Only OpenAI-compatible providers can be routed with --api-base; for
-		// those, hand the key over via env (never the command line).
+		// those, hand the key over via env (never the command line). The key is
+		// resolved through the harness credential seam when available (values may
+		// live encrypted in .credentials.yaml rather than in the process env),
+		// falling back to the plain environment variable.
 		if (provider && provider.api === "openai-completions" && provider.baseURL) {
 			args.push("--api-base", provider.baseURL);
-			if (provider.keyEnv && process.env[provider.keyEnv]) {
-				env.LIVEBENCH_API_KEY = process.env[provider.keyEnv];
+			if (provider.keyEnv) {
+				let key;
+				try {
+					const credentials = ctx.get ? ctx.get("credentials") : undefined;
+					if (credentials && typeof credentials.resolve === "function") {
+						const resolved = await credentials.resolve(provider.keyEnv);
+						if (resolved && typeof resolved.value === "string" && resolved.value.length > 0) key = resolved.value;
+					}
+				} catch { /* credential seam unavailable — fall through to env */ }
+				if (!key && process.env[provider.keyEnv]) key = process.env[provider.keyEnv];
+				if (key) env.LIVEBENCH_API_KEY = key;
 			}
 		}
 
@@ -486,6 +563,7 @@ function apply(ctx) {
 					id,
 					name: pname,
 					routable: api === "openai-completions" && typeof baseURL === "string" && baseURL.length > 0,
+					baseURL: baseURL ?? null,
 					models,
 				})),
 			});
@@ -515,7 +593,7 @@ function apply(ctx) {
 				sendJson(res, 409, { ok: false, error: "another evaluation is already running" });
 				return;
 			}
-			const result = startRun(body);
+			const result = await startRun(body);
 			sendJson(res, result.status, result.payload);
 		},
 	}), `${name}: start route`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-livebench-panel",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "DSH web plugin: a LiveBench tab in the Trajectory view (right of 对话/轨迹). Run LiveBench evaluations against every model configured in the DeepSeek Harness — pick provider/model, category, task, release and question range from dropdowns, watch progress, and read scores in place.",
   "license": "MIT",
   "type": "module",