@workbench-ai/workbench 0.0.47 → 0.0.49

package/dist/dev-open-server.d.ts

@@ -22,6 +22,7 @@ export declare function localBenchmarkSnapshot(context: LocalWorkbenchRequestCon
     currentBenchmarkFingerprint: string | null;
     summaries: {
         id: string;
+        name?: string;
         ordinal: number;
         benchmarkFingerprint: string;
         subjectFingerprint: string;
@@ -42,6 +43,7 @@ export declare function localBenchmarkSnapshot(context: LocalWorkbenchRequestCon
         benchmarkFingerprint: string;
         subjectFingerprint: string;
         subjectId: string;
+        subjectName?: string;
         createdAt: string;
         updatedAt: string;
         status: import("@workbench-ai/workbench-contract").EvaluationStatus;

package/dist/dev-open-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dev-open-server.d.ts","sourceRoot":"","sources":["../src/dev-open-server.ts"],"names":[],"mappings":"AAKA,OAAO,EAUL,KAAK,mBAAmB,EAIzB,MAAM,8BAA8B,CAAC;AAatC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAkBD,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACtD;AAKD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAwClC;AAoOD,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiBjF;AAUD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,uFAiCrC;AAwBD,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAExF;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GACnC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC"}
+{"version":3,"file":"dev-open-server.d.ts","sourceRoot":"","sources":["../src/dev-open-server.ts"],"names":[],"mappings":"AAKA,OAAO,EAUL,KAAK,mBAAmB,EAIzB,MAAM,8BAA8B,CAAC;AAatC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAkBD,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACtD;AAKD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAwClC;AAoOD,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiBjF;AAUD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,uFAiCrC;AAwBD,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAExF;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GACnC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC"}

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAsHA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA6BD,UAAU,iBAAiB;CAAG;AAwJ9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CA8GjB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAsHA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA6BD,UAAU,iBAAiB;CAAG;AAqK9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CA8GjB"}

package/dist/index.js

@@ -22,6 +22,16 @@ function getCliVersion() {
     const manifest = require("../package.json");
     return typeof manifest.version === "string" ? manifest.version : "unknown";
 }
+class WorkbenchApiRequestError extends Error {
+    status;
+    body;
+    constructor(status, message, body) {
+        super(message);
+        this.name = "WorkbenchApiRequestError";
+        this.status = status;
+        this.body = body;
+    }
+}
 const DEFAULT_BASE_URL = "https://v2.workbench.ai";
 export async function runCli(argv, io = {
     stdin: process.stdin,
@@ -2358,52 +2368,55 @@ async function pushBenchmark(argv, io) {
         throw new UsageError("Missing hosted benchmark. Run workbench push from a source directory.");
     }
     if (!origin.writable) {
-        const upstream = upstreamFromOrigin(origin);
-        if (dryRun) {
+        const signedInUsername = dryRun ? null : await readAuthenticatedWorkbenchUsername(baseUrl);
+        if (signedInUsername !== origin.owner) {
+            const upstream = upstreamFromOrigin(origin);
+            if (dryRun) {
+                writeOutput({
+                    ok: true,
+                    dryRun: true,
+                    action: "create",
+                    dir,
+                    baseUrl,
+                    benchmarkName: source.spec.name,
+                    tag: asOptionalString(parsed.flags.tag) ?? null,
+                    visibility,
+                    sourceFileCount: sourceFileCount(source),
+                    upstream: upstream ?? null,
+                }, parsed, io, () => `Would create a writable benchmark from read-only origin ${origin.owner}/${origin.project}.`);
+                return 0;
+            }
+            const { project, publishedProject, origin: nextOrigin } = await createHostedBenchmarkFromSource({
+                baseUrl,
+                dir,
+                source,
+                visibility,
+                upstream,
+            });
             writeOutput({
                 ok: true,
-                dryRun: true,
                 action: "create",
-                dir,
-                baseUrl,
-                benchmarkName: source.spec.name,
+                benchmark: publishedProject,
                 tag: asOptionalString(parsed.flags.tag) ?? null,
                 visibility,
-                sourceFileCount: sourceFileCount(source),
+                origin: nextOrigin,
                 upstream: upstream ?? null,
-            }, parsed, io, () => `Would create a writable benchmark from read-only origin ${origin.owner}/${origin.project}.`);
+                urls: buildWorkbenchResourceUrls({
+                    baseUrl,
+                    projectId: publishedProject.id ?? project.id,
+                    owner: nextOrigin.owner,
+                    projectName: nextOrigin.project,
+                }),
+            }, parsed, io, (record) => {
+                const value = record;
+                return [
+                    `Pushed ${value.origin.owner}/${value.origin.project} (${value.origin.projectId}).`,
+                    ...(value.upstream ? [`Upstream: ${value.upstream.owner}/${value.upstream.project}`] : []),
+                    `Open benchmark: ${value.urls.benchmark}`,
+                ].join("\n");
+            });
             return 0;
         }
-        const { project, publishedProject, origin: nextOrigin } = await createHostedBenchmarkFromSource({
-            baseUrl,
-            dir,
-            source,
-            visibility,
-            upstream,
-        });
-        writeOutput({
-            ok: true,
-            action: "create",
-            benchmark: publishedProject,
-            tag: asOptionalString(parsed.flags.tag) ?? null,
-            visibility,
-            origin: nextOrigin,
-            upstream: upstream ?? null,
-            urls: buildWorkbenchResourceUrls({
-                baseUrl,
-                projectId: publishedProject.id ?? project.id,
-                owner: nextOrigin.owner,
-                projectName: nextOrigin.project,
-            }),
-        }, parsed, io, (record) => {
-            const value = record;
-            return [
-                `Pushed ${value.origin.owner}/${value.origin.project} (${value.origin.projectId}).`,
-                ...(value.upstream ? [`Upstream: ${value.upstream.owner}/${value.upstream.project}`] : []),
-                `Open benchmark: ${value.urls.benchmark}`,
-            ].join("\n");
-        });
-        return 0;
     }
     if (dryRun) {
         writeOutput({
@@ -2480,6 +2493,11 @@ async function createHostedBenchmarkFromSource(args) {
     });
     return { project, publishedProject, origin };
 }
+async function readAuthenticatedWorkbenchUsername(baseUrl) {
+    const config = await loadConfig();
+    const status = await readWorkbenchProfileStatus({ ...config, baseUrl });
+    return status.authenticated ? status.profile?.username ?? null : null;
+}
 function upstreamFromOrigin(origin) {
     if (!origin.owner || !origin.project || !origin.projectId || !origin.sourceRevisionId) {
         return undefined;
@@ -2824,15 +2842,20 @@ async function startHostedWorkflow(workflow, argv, io) {
 }
 async function ensureHostedImproveBaseSubject(args) {
     if (args.subjectId) {
-        const response = await apiRequest(projectApiPath(args.target.projectId, "/subjects"), {}, args.target.baseUrl);
-        const subject = response.subjects.find((entry) => entry.id === args.subjectId);
+        const subject = await readHostedSubjectSummary(args.target, args.subjectId);
         if (!subject) {
             throw new UsageError(`Base subject ${args.subjectId} was not found for the current benchmark.`);
         }
-        if (subject && (subject.status === "evaluated" || subject.eval != null)) {
+        if (hostedSubjectIsEvaluated(subject)) {
             return args.subjectId;
         }
     }
+    else {
+        const activeSubject = await readEvaluatedActiveHostedSubject(args.target);
+        if (activeSubject) {
+            return activeSubject.id;
+        }
+    }
     const response = await apiRequest(projectApiPath(args.target.projectId, "/runs"), {
         method: "POST",
         body: {
@@ -2856,6 +2879,22 @@ async function ensureHostedImproveBaseSubject(args) {
     }
     return watched.subjectId;
 }
+async function readHostedSubjectSummary(target, subjectId) {
+    const response = await apiRequest(projectApiPath(target.projectId, "/subjects"), {}, target.baseUrl);
+    return response.subjects.find((entry) => entry.id === subjectId) ?? null;
+}
+async function readEvaluatedActiveHostedSubject(target) {
+    const response = await apiRequest(projectApiPath(target.projectId), {}, target.baseUrl);
+    const activeSubjectId = response.benchmark.activeSubjectId;
+    if (!activeSubjectId) {
+        return null;
+    }
+    const subject = await readHostedSubjectSummary(target, activeSubjectId);
+    return subject && hostedSubjectIsEvaluated(subject) ? subject : null;
+}
+function hostedSubjectIsEvaluated(subject) {
+    return subject.status === "evaluated" || subject.eval != null;
+}
 async function benchmarkList(argv, io) {
     const parsed = parseArgs(argv);
     rejectUnknownFlags(parsed, new Set(["json"]));
@@ -3475,7 +3514,20 @@ async function watchHostedRun(args) {
     const deadline = args.timeoutMs === undefined ? undefined : Date.now() + args.timeoutMs;
     let lastRun = null;
     while (true) {
-        const response = await apiRequest(projectApiPath(args.target.projectId, `/runs/${encodeURIComponent(args.runId)}`), {}, args.target.baseUrl);
+        let response;
+        try {
+            response = await apiRequest(projectApiPath(args.target.projectId, `/runs/${encodeURIComponent(args.runId)}`), {}, args.target.baseUrl);
+        }
+        catch (error) {
+            if (isTransientApiRequestError(error)) {
+                if (deadline !== undefined && Date.now() > deadline) {
+                    throw new Error(`Timed out waiting for run ${args.runId}; last status was ${lastRun?.status ?? "unknown"} and the latest poll failed with ${error.message}.`);
+                }
+                await sleep(args.intervalMs);
+                continue;
+            }
+            throw error;
+        }
         lastRun = response.run;
         if (response.run.status === "finished") {
             return response.run;
@@ -3753,8 +3805,8 @@ async function apiRequest(apiPath, options = {}, baseUrlOverride) {
     });
     if (!response.ok) {
         const text = await response.text();
-        throw new Error(readResponseError(text) ||
-            `Request failed with status ${response.status}.`);
+        throw new WorkbenchApiRequestError(response.status, readResponseError(text) ||
+            `Request failed with status ${response.status}${response.statusText ? ` ${response.statusText}` : ""}.`, text);
     }
     return (await response.json());
 }
@@ -3853,9 +3905,17 @@ function readResponseError(text) {
                 : "";
     }
     catch {
-        return text;
+        const trimmed = text.trim();
+        if (trimmed.startsWith("<")) {
+            return "";
+        }
+        return trimmed;
     }
 }
+function isTransientApiRequestError(error) {
+    return error instanceof WorkbenchApiRequestError
+        && (error.status === 408 || error.status === 429 || error.status >= 500);
+}
 function readOAuthError(text) {
     try {
         const body = JSON.parse(text);

package/dist/init-template-pack.js

@@ -84,6 +84,7 @@ function skillBenchmarkSpec(name, agent) {
         "        parallelism: 2",
         "        judge:",
         `          use: ${agent}`,
+        ...agentDefaultWithLines(agent, "          "),
         "        criteria:",
         "          - id: task_fit",
         "            description: The response follows the task prompt and uses the skill's workflow.",
@@ -104,6 +105,7 @@ function skillSubjectSpec(name, agent) {
         "  command: sh input/subject/prepare.sh",
         "run:",
         `  use: ${agent}`,
+        ...agentDefaultWithLines(agent, "  "),
         "",
     ].join("\n");
 }
@@ -116,9 +118,19 @@ function optimizerSpec(name, editablePath, agent) {
         `  - ${editablePath}`,
         "improve:",
         `  use: ${agent}`,
+        ...agentDefaultWithLines(agent, "  "),
         "",
     ].join("\n");
 }
+function agentDefaultWithLines(agent, indent) {
+    if (agent !== "codex") {
+        return [];
+    }
+    return [
+        `${indent}with:`,
+        `${indent}  model: gpt-5.5`,
+    ];
+}
 function commandBenchmarkSpec(name) {
     return [
         "version: 3",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workbench-ai/workbench",
-  "version": "0.0.47",
+  "version": "0.0.49",
   "type": "module",
   "repository": {
     "type": "git",
@@ -21,9 +21,9 @@
   ],
   "dependencies": {
     "yaml": "^2.8.2",
-    "@workbench-ai/workbench-protocol": "0.0.47",
-    "@workbench-ai/workbench-built-in-adapters": "0.0.47",
-    "@workbench-ai/workbench-core": "0.0.47"
+    "@workbench-ai/workbench-built-in-adapters": "0.0.49",
+    "@workbench-ai/workbench-protocol": "0.0.49",
+    "@workbench-ai/workbench-core": "0.0.49"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.2.2",