@workbench-ai/workbench 0.0.80 → 0.0.82

package/dist/index.js

@@ -348,11 +348,11 @@ export async function runCli(argv, io = {
             });
             const artifactIds = await artifactIdsByRunId(core, runs);
             const failedRuns = runs.filter((run) => run.status === "failed" || run.status === "canceled");
+            const coverage = await evalCoverageSummaries(core, runs);
+            const deltas = await evalDeltas(core, runs);
             if (failedRuns.length > 0) {
-                return emitEvalFailure(runs, failedRuns, artifactIds, parsed, io);
+                return emitEvalFailure(runs, failedRuns, artifactIds, coverage, deltas, parsed, io);
             }
-            const deltas = await evalDeltas(core, runs);
-            const coverage = await evalCoverageSummaries(core, runs);
             const next = await evalSuccessNextCommand(core, runs);
             return emitResult("workbench.cli.eval.v1", {
                 result: runs.map((run) => runSummary(run, artifactIds.get(run.id) ?? [])),
@@ -361,8 +361,8 @@ export async function runCli(argv, io = {
                 next: next,
             }, parsed, io, () => [
                 runs.map(formatRun).join("\n"),
-                ...coverage.map(formatEvalCoverage),
-                ...deltas.map(formatEvalDelta),
+                ...formatEvalCoverageLines(coverage),
+                ...formatEvalDeltaLines(deltas),
                 ...(next ? [`next: ${next}`] : []),
             ].filter(Boolean).join("\n"));
         }
@@ -947,20 +947,23 @@ async function handleCloudEval(parsed, io) {
         return 130;
     }
     const failedRuns = started.runs.filter((run) => run.status === "failed" || run.status === "canceled");
+    const coverage = await evalCoverageSummaries(started.core, started.runs);
+    const deltas = await evalDeltas(started.core, started.runs);
     if (failedRuns.length > 0) {
-        return emitEvalFailure(started.runs, failedRuns, artifactIds, parsed, io);
+        return emitEvalFailure(started.runs, failedRuns, artifactIds, coverage, deltas, parsed, io);
     }
-    const deltas = await evalDeltas(started.core, started.runs);
     const next = await evalSuccessNextCommand(started.core, started.runs);
     return emitResult("workbench.cli.eval.v1", {
         result: started.runs.map((run) => runSummary(run, artifactIds.get(run.id) ?? [])),
+        coverage: coverage,
         deltas: deltas,
         next: next,
         cloud: cloudExecutionSummary(started),
     }, parsed, io, () => [
         `Completed hosted eval on ${started.remote.url}.`,
         started.runs.map(formatRun).join("\n"),
-        ...deltas.map(formatEvalDelta),
+        ...formatEvalCoverageLines(coverage),
+        ...formatEvalDeltaLines(deltas),
         ...(next ? [`next: ${next}`] : []),
     ].filter(Boolean).join("\n"));
 }
@@ -1124,14 +1127,19 @@ async function startCloudExecution(command, parsed, io) {
                 samples: intFlag(parsed, "samples"),
                 budget: intFlag(parsed, "budget"),
             }));
-        writeCloudProgress(io, "workbench cloud: checking provider auth.", showProgress);
-        await cloudPreScheduleStep(command, interrupt, preflightCloudAdapterAuth({
+        const adapterAuthTargets = await cloudPreScheduleStep(command, interrupt, resolveCloudAdapterAuthTargets({
             root,
             versionId: request.versionId,
             parsed,
-            baseUrl: source.baseUrl,
             authToken: token,
         }));
+        if (adapterAuthTargets.length > 0) {
+            writeCloudProgress(io, "workbench cloud: checking provider auth.", showProgress);
+            await cloudPreScheduleStep(command, interrupt, assertCloudAdapterAuthConnected({
+                baseUrl: source.baseUrl,
+                targets: adapterAuthTargets,
+            }));
+        }
         writeCloudProgress(io, "workbench cloud: syncing source to cloud.", showProgress);
         const syncBefore = await cloudPreScheduleStep(command, interrupt, syncWorkbenchRemote({ ...core, remote: remote.name }));
         writeCloudProgress(io, `workbench cloud: scheduling hosted ${command}.`, showProgress);
@@ -1230,7 +1238,7 @@ function cloudCanceledBeforeRunIdError(command) {
         exitCode: 130,
     });
 }
-async function preflightCloudAdapterAuth(input) {
+async function resolveCloudAdapterAuthTargets(input) {
     const snapshot = await createWorkbenchReadOnlyInspectionSnapshot({ dir: input.root, authToken: input.authToken });
     const version = snapshotVersionByRef(snapshot, input.versionId);
     if (!version) {
@@ -1245,7 +1253,10 @@ async function preflightCloudAdapterAuth(input) {
         agent: stringFlag(input.parsed, "agents"),
         authToken: input.authToken,
     });
-    const targets = uniqueAdapterAuthTargets(runtime.selectedAgents.flatMap(cloudAdapterAuthTargetsForAgent));
+    return uniqueAdapterAuthTargets(runtime.selectedAgents.flatMap(cloudAdapterAuthTargetsForAgent));
+}
+async function assertCloudAdapterAuthConnected(input) {
+    const targets = uniqueAdapterAuthTargets(input.targets);
     if (targets.length === 0) {
         return;
     }
@@ -2728,7 +2739,7 @@ async function artifactIdsByRunId(core, runs) {
     }
     return byRun;
 }
-function emitEvalFailure(runs, failedRuns, artifactIds, parsed, io) {
+function emitEvalFailure(runs, failedRuns, artifactIds, coverage, deltas, parsed, io) {
     const next = evalFailureNextCommand(failedRuns);
     if (parsed.flags.json === true) {
         io.stdout.write(`${JSON.stringify({
@@ -2740,13 +2751,17 @@ function emitEvalFailure(runs, failedRuns, artifactIds, parsed, io) {
             evidenceSaved: true,
             runs: runs.map((run) => runFailureSummary(run, artifactIds.get(run.id) ?? [])),
             failedRuns: failedRuns.map((run) => runFailureSummary(run, artifactIds.get(run.id) ?? [])),
+            coverage: coverage,
+            deltas: deltas,
             next,
         }, null, 2)}\n`);
         return 1;
     }
     io.stdout.write([
         "Eval failed; evidence was saved.",
-        ...failedRuns.map(formatRun),
+        runs.map(formatRun).join("\n"),
+        ...formatEvalCoverageLines(coverage),
+        ...formatEvalDeltaLines(deltas),
         ...(next ? [`next: ${next}`] : []),
     ].join("\n") + "\n");
     return 1;
@@ -3317,6 +3332,8 @@ async function evalCoverageSummaries(core, runs) {
         const samples = new Set(jobs.map((job) => `${job.caseId}\0${job.sample}`));
         return {
             runId: run.id,
+            skillName: run.skillName,
+            agentName: run.agentName,
             cases: cases.size,
             samples: samples.size,
             jobs: jobs.length,
@@ -3325,12 +3342,19 @@ async function evalCoverageSummaries(core, runs) {
         };
     });
 }
-function formatEvalCoverage(coverage) {
+function formatEvalCoverageLines(coverage) {
+    const includeRunLabels = coverage.length > 1;
+    return coverage.map((entry) => formatEvalCoverage(entry, includeRunLabels));
+}
+function formatEvalCoverage(coverage, includeRunLabels = false) {
     return [
         `coverage cases=${coverage.cases}`,
         `samples=${coverage.samples}`,
         `jobs=${coverage.jobs}`,
         coverage.failed > 0 ? `failed=${coverage.failed}` : undefined,
+        includeRunLabels ? `run=${displayRef(coverage.runId)}` : undefined,
+        includeRunLabels ? `skill=${coverage.skillName}` : undefined,
+        includeRunLabels ? `agent=${coverage.agentName}` : undefined,
     ].filter(Boolean).join(" ");
 }
 async function evalDeltas(core, runs) {
@@ -3356,16 +3380,23 @@ async function evalDeltas(core, runs) {
         };
     });
 }
-function formatEvalDelta(delta) {
+function formatEvalDeltaLines(deltas) {
+    const includeRunLabels = deltas.length > 1;
+    return deltas
+        .map((delta) => formatEvalDelta(delta, includeRunLabels))
+        .filter((line) => line.length > 0);
+}
+function formatEvalDelta(delta, includeRunLabels = false) {
     if (delta.score === undefined) {
         return "";
     }
+    const label = includeRunLabels ? `${delta.skillName}/${delta.agentName}` : delta.skillName;
     const score = delta.score.toFixed(3);
     if (delta.previousScore === undefined || delta.delta === undefined) {
-        return `${delta.skillName} ${displayRef(delta.versionId)} ${score}`;
+        return `${label} ${displayRef(delta.versionId)} ${score}`;
     }
     const sign = delta.delta >= 0 ? "+" : "";
-    return `${delta.skillName} ${displayRef(delta.versionId)} ${score} (was ${delta.previousScore.toFixed(3)}, ${sign}${delta.delta.toFixed(3)})`;
+    return `${label} ${displayRef(delta.versionId)} ${score} (was ${delta.previousScore.toFixed(3)}, ${sign}${delta.delta.toFixed(3)})`;
 }
 async function evalSuccessNextCommand(core, runs) {
     if (runs.length === 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workbench-ai/workbench",
-  "version": "0.0.80",
+  "version": "0.0.82",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/workbench-ai/workbench.git",
@@ -22,10 +22,10 @@
   "dependencies": {
     "skills": "1.5.11",
     "yaml": "^2.8.2",
-    "@workbench-ai/workbench-built-in-adapters": "0.0.80",
-    "@workbench-ai/workbench-protocol": "0.0.80",
-    "@workbench-ai/workbench-contract": "0.0.80",
-    "@workbench-ai/workbench-core": "0.0.80"
+    "@workbench-ai/workbench-built-in-adapters": "0.0.82",
+    "@workbench-ai/workbench-contract": "0.0.82",
+    "@workbench-ai/workbench-protocol": "0.0.82",
+    "@workbench-ai/workbench-core": "0.0.82"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.2.2",
@@ -36,7 +36,7 @@
     "react-dom": "^19.2.0",
     "typescript": "^5.9.2",
     "vitest": "^3.2.4",
-    "@workbench-ai/workbench-ui": "0.0.80"
+    "@workbench-ai/workbench-ui": "0.0.82"
   },
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.json && chmod 755 dist/workbench.js && node ./scripts/build-dev-open-assets.mjs",