@workbench-ai/workbench 0.0.49 → 0.0.51

package/dist/dev-open-server.d.ts

@@ -1,4 +1,4 @@
-import { type SurfaceSnapshotFile } from "@workbench-ai/workbench-core";
+import { type RunSummary, type SurfaceSnapshotFile } from "@workbench-ai/workbench-core";
 import { type LocalProjectSource } from "./project-source.js";
 export interface LocalWorkbenchDevServer {
     url: string;
@@ -20,30 +20,17 @@ export declare function localBenchmarkSnapshot(context: LocalWorkbenchRequestCon
     workspaceRoot: string;
     activeId: string | null;
     currentBenchmarkFingerprint: string | null;
-    summaries: {
-        id: string;
-        name?: string;
-        ordinal: number;
-        benchmarkFingerprint: string;
-        subjectFingerprint: string;
-        ownerUserId?: string;
-        ownerUsername?: string;
-        visibility?: "private" | "public";
-        createdAt: string;
-        baseId?: string;
-        referenceIds: string[];
-        status: import("@workbench-ai/workbench-contract").SubjectStatus;
-        fileChanges: string[];
-        metrics?: Record<string, number>;
-        usage?: import("@workbench-ai/workbench-contract").UsageSummary;
-    }[];
+    summaries: import("@workbench-ai/workbench-contract").CandidateSummary[];
     evaluations: {
         id: string;
         runId: string;
         benchmarkFingerprint: string;
-        subjectFingerprint: string;
-        subjectId: string;
-        subjectName?: string;
+        candidateFingerprint: string;
+        candidateId: string;
+        candidateName?: string;
+        candidateVersion: number;
+        candidateRunId?: string;
+        candidateRunName?: string;
         createdAt: string;
         updatedAt: string;
         status: import("@workbench-ai/workbench-contract").EvaluationStatus;
@@ -51,11 +38,14 @@ export declare function localBenchmarkSnapshot(context: LocalWorkbenchRequestCon
         completedSampleCount: number;
         errorSampleCount: number;
         metrics?: Record<string, import("@workbench-ai/workbench-contract").MetricStats>;
+        selectionMetric?: string;
+        selectionLabel?: string;
+        selectionScore?: import("@workbench-ai/workbench-contract").MetricStats;
         durationMs?: import("@workbench-ai/workbench-contract").MetricStats;
         usage?: import("@workbench-ai/workbench-contract").EvaluationUsageStats;
         error?: string;
     }[];
-    runs: import("@workbench-ai/workbench-contract").RunSummary[];
+    runs: RunSummary[];
 }>;
 export declare function localSpecDocument(context: LocalWorkbenchRequestContext, benchmarkFingerprint?: string | null): Promise<import("@workbench-ai/workbench-contract").AuthoredWorkbenchSourceDocument>;
 export declare function localSourceFiles(workspace: string): Promise<SurfaceSnapshotFile[]>;

package/dist/dev-open-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dev-open-server.d.ts","sourceRoot":"","sources":["../src/dev-open-server.ts"],"names":[],"mappings":"AAKA,OAAO,EAUL,KAAK,mBAAmB,EAIzB,MAAM,8BAA8B,CAAC;AAatC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAkBD,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACtD;AAKD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAwClC;AAoOD,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiBjF;AAUD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,uFAiCrC;AAwBD,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAExF;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GACnC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC"}
+{"version":3,"file":"dev-open-server.d.ts","sourceRoot":"","sources":["../src/dev-open-server.ts"],"names":[],"mappings":"AAKA,OAAO,EAYL,KAAK,UAAU,EACf,KAAK,mBAAmB,EAIzB,MAAM,8BAA8B,CAAC;AAatC,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,uBAAuB;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,MAAM,WAAW,8BAA8B;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAkBD,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACtD;AAKD,wBAAsB,4BAA4B,CAChD,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAwClC;AAoQD,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiBjF;AAeD,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,uFAiCrC;AAwBD,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAExF;AAED,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,4BAA4B,EACrC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,GACnC,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAchC"}

package/dist/dev-open-server.js

@@ -2,8 +2,8 @@ import { promises as fs } from "node:fs";
 import http from "node:http";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
-import { buildSubjectCaseExecutionRefs, buildWorkbenchExecutionEvidence, createSubjectFilePreview, createCaseReview, loadAuthoredWorkbenchSourceDocument, summarizeSubjectFiles, traceSessionLabel, } from "@workbench-ai/workbench-core";
-import { readLocalExecutionFiles, loadLocalArchiveIndex, readLocalEvaluationRecord, readLocalJobInRun, readLocalRunJobs, readLocalSubjectFilesForId, readLocalSubjectRecord, } from "./local-archive.js";
+import { buildCandidateCaseExecutionRefs, buildWorkbenchExecutionEvidence, candidateRecordWithoutDerivedFields, candidateSummaryFromRecord, createCandidateFilePreview, createCaseReview, loadAuthoredWorkbenchSourceDocument, summarizeCandidateFiles, traceSessionLabel, } from "@workbench-ai/workbench-core";
+import { readLocalExecutionFiles, loadLocalArchiveIndex, readLocalEvaluationRecord, readLocalJobInRun, readLocalRunJobs, readLocalCandidateFilesForId, readLocalCandidateRecord, } from "./local-archive.js";
 import { readLocalAuthoredProjectSource, readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
 import { localBenchmarkFingerprint } from "./benchmark-fingerprint.js";
 class LocalApiError extends Error {
@@ -108,7 +108,34 @@ async function handleLocalWorkbenchRequest(args) {
         await sendFontFile(args.response, args.context.assetsRoot, url, args.request.method);
         return;
     }
-    await sendHtml(args.response, args.request.method);
+    if (url.pathname.startsWith("/assets/")) {
+        throw new LocalApiError("Workbench local asset not found.", 404);
+    }
+    await sendHtml(args.response, args.request.method, isKnownWorkbenchDocumentPath(url.pathname) ? 200 : 404);
+}
+function isKnownWorkbenchDocumentPath(pathname) {
+    const segments = pathname.split("/").filter(Boolean).map(decodeDocumentPathSegment);
+    if (segments.length === 0) {
+        return true;
+    }
+    if (segments[0] === "evaluations") {
+        return segments.length === 1;
+    }
+    if (segments[0] !== "candidates") {
+        return false;
+    }
+    if (segments.length === 1 || segments.length === 2) {
+        return true;
+    }
+    return segments.length === 3 && (segments[2] === "files" || segments[2] === "manifest");
+}
+function decodeDocumentPathSegment(segment) {
+    try {
+        return decodeURIComponent(segment);
+    }
+    catch {
+        return segment;
+    }
 }
 async function handleApiRequest(request, response, context, url) {
     const { workspace } = context;
@@ -120,45 +147,45 @@ async function handleApiRequest(request, response, context, url) {
             sendJson(response, await localSpecDocument(context, readOptionalSearchString(url.searchParams, "fingerprint")), 200, request.method);
             return;
         case "/api/source/files":
-            sendJson(response, summarizeSubjectFiles(await localBenchmarkMountedFiles(context, readOptionalSearchString(url.searchParams, "fingerprint")), []), 200, request.method);
+            sendJson(response, summarizeCandidateFiles(await localBenchmarkMountedFiles(context, readOptionalSearchString(url.searchParams, "fingerprint")), []), 200, request.method);
             return;
         case "/api/source/preview":
-            sendJson(response, createSubjectFilePreview({
+            sendJson(response, createCandidateFilePreview({
                 files: await localBenchmarkMountedFiles(context, readOptionalSearchString(url.searchParams, "fingerprint")),
                 path: readSearchString(url.searchParams, "path"),
                 view: readPreviewMode(url.searchParams),
             }), 200, request.method);
             return;
         case "/api/record":
-            sendJson(response, await readSubjectForApi(workspace, readSearchString(url.searchParams, "id")), 200, request.method);
+            sendJson(response, await readCandidateForApi(workspace, readSearchString(url.searchParams, "id")), 200, request.method);
             return;
         case "/api/evaluation":
             sendJson(response, await readEvaluationForApi(workspace, readSearchString(url.searchParams, "id")), 200, request.method);
             return;
-        case "/api/subject/files": {
-            const subjectId = readSearchString(url.searchParams, "id");
-            const subject = await readSubjectForApi(workspace, subjectId);
-            sendJson(response, summarizeSubjectFiles(await readSubjectFilesForApi(workspace, subjectId), subject.fileChanges), 200, request.method);
+        case "/api/candidate/files": {
+            const candidateId = readSearchString(url.searchParams, "id");
+            const candidate = await readCandidateForApi(workspace, candidateId);
+            sendJson(response, summarizeCandidateFiles(await readCandidateFilesForApi(workspace, candidateId), candidate.fileChanges), 200, request.method);
             return;
         }
-        case "/api/subject/preview": {
-            const subjectId = readSearchString(url.searchParams, "id");
-            sendJson(response, createSubjectFilePreview({
-                files: await readSubjectFilesForApi(workspace, subjectId),
+        case "/api/candidate/preview": {
+            const candidateId = readSearchString(url.searchParams, "id");
+            sendJson(response, createCandidateFilePreview({
+                files: await readCandidateFilesForApi(workspace, candidateId),
                 path: readSearchString(url.searchParams, "path"),
                 view: readPreviewMode(url.searchParams),
             }), 200, request.method);
             return;
         }
         case "/api/case-review": {
-            const subjectId = readSearchString(url.searchParams, "id");
+            const candidateId = readSearchString(url.searchParams, "id");
             const caseId = readSearchString(url.searchParams, "case");
             const runId = readSearchString(url.searchParams, "run");
             const jobs = await readLocalRunJobs(workspace, runId);
             sendJson(response, createCaseReview({
-                subject: await readSubjectForApi(workspace, subjectId),
+                candidate: await readCandidateForApi(workspace, candidateId),
                 caseId,
-                executions: buildSubjectCaseExecutionRefs({ jobs, subjectId, caseId }),
+                executions: buildCandidateCaseExecutionRefs({ jobs, candidateId, caseId }),
             }), 200, request.method);
             return;
         }
@@ -190,7 +217,7 @@ async function handleApiRequest(request, response, context, url) {
             const previewJobId = readSearchString(url.searchParams, "id");
             const previewFilePath = readSearchString(url.searchParams, "path");
             const previewFiles = await readExecutionFilesForRun(workspace, previewRunId, previewJobId);
-            sendJson(response, createSubjectFilePreview({
+            sendJson(response, createCandidateFilePreview({
                 files: previewFiles,
                 path: previewFilePath,
                 view: readPreviewMode(url.searchParams),
@@ -204,11 +231,11 @@ async function handleApiRequest(request, response, context, url) {
 export async function localBenchmarkSnapshot(context) {
     const { workspace } = context;
     const snapshot = await loadLocalArchiveIndex(workspace);
-    const subjects = snapshot.subjects.filter(isInspectableSubjectRecord);
-    const summaries = subjects.map(subjectSummary);
-    const activeId = snapshot.activeId && subjects.some((subject) => subject.id === snapshot.activeId)
+    const candidates = snapshot.candidates.filter(isInspectableCandidateRecord);
+    const summaries = candidates.map(candidateSummary);
+    const activeId = snapshot.activeId && candidates.some((candidate) => candidate.id === snapshot.activeId)
         ? snapshot.activeId
-        : subjects.at(-1)?.id ?? null;
+        : candidates.at(-1)?.id ?? null;
     const currentBenchmarkFingerprint = await readCurrentBenchmarkFingerprint(context);
     return {
         workspaceRoot: path.resolve(workspace),
@@ -216,9 +243,13 @@ export async function localBenchmarkSnapshot(context) {
         currentBenchmarkFingerprint,
         summaries,
         evaluations: snapshot.evaluations.map(evaluationSummary),
-        runs: snapshot.runs,
+        runs: snapshot.runs.map(publicLocalRunSummary),
     };
 }
+function publicLocalRunSummary(run) {
+    const { executionFingerprint: _executionFingerprint, ...summary } = run;
+    return summary;
+}
 async function readCurrentBenchmarkFingerprint(context) {
     return await context.readProjectSource()
         .then(localBenchmarkFingerprint)
@@ -290,8 +321,8 @@ export async function localBenchmarkMountedFiles(context, benchmarkFingerprint)
     return inspectableEngineCaseFiles(projectSource.engineCases);
 }
 function localHistoricalBenchmarkDocument(snapshot, benchmarkFingerprint) {
-    const subject = snapshot.subjects.find((entry) => entry.benchmarkFingerprint === benchmarkFingerprint && readBenchmarkSourceMetadata(entry));
-    const source = subject ? readBenchmarkSourceMetadata(subject) : null;
+    const candidate = snapshot.candidates.find((entry) => entry.benchmarkFingerprint === benchmarkFingerprint && readBenchmarkSourceMetadata(entry));
+    const source = candidate ? readBenchmarkSourceMetadata(candidate) : null;
     if (!source?.sourceYaml) {
         return null;
     }
@@ -317,28 +348,28 @@ function engineCaseFiles(bundle) {
         ? buckets.source
         : [...(buckets.public ?? []), ...(buckets.private ?? [])];
 }
-function subjectSummary(subject) {
-    const { eval: _eval, prompt: _prompt, meta: _meta, ...summary } = subject;
-    return summary;
+function candidateSummary(candidate) {
+    return candidateSummaryFromRecord(candidate);
 }
-function isInspectableSubjectRecord(subject) {
-    return Boolean(subject.eval || asRecord(asRecord(subject.meta)?.source));
+function isInspectableCandidateRecord(candidate) {
+    return Boolean(candidate.eval || asRecord(asRecord(candidate.meta)?.source));
 }
 function evaluationSummary(evaluation) {
     const { evaluation: _evaluation, ...summary } = evaluation;
     return summary;
 }
-async function readSubjectForApi(workspace, subjectId) {
-    return await readArchiveRecord("Subject", subjectId, () => readLocalSubjectRecord(workspace, subjectId));
+async function readCandidateForApi(workspace, candidateId) {
+    const candidate = await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateRecord(workspace, candidateId));
+    return candidateRecordWithoutDerivedFields(candidate);
 }
 async function readEvaluationForApi(workspace, evaluationId) {
     return await readArchiveRecord("Evaluation", evaluationId, () => readLocalEvaluationRecord(workspace, evaluationId));
 }
-async function readSubjectFilesForApi(workspace, subjectId) {
-    return await readArchiveRecord("Subject", subjectId, () => readLocalSubjectFilesForId(workspace, subjectId));
+async function readCandidateFilesForApi(workspace, candidateId) {
+    return await readArchiveRecord("Candidate", candidateId, () => readLocalCandidateFilesForId(workspace, candidateId));
 }
-function readBenchmarkSourceMetadata(subject) {
-    const benchmark = asRecord(asRecord(subject.meta)?.benchmark);
+function readBenchmarkSourceMetadata(candidate) {
+    const benchmark = asRecord(asRecord(candidate.meta)?.benchmark);
     const files = Array.isArray(benchmark?.files)
         ? benchmark.files
             .map(readSurfaceSnapshotFile)
@@ -390,7 +421,7 @@ async function readArchiveRecord(kind, id, read) {
 }
 async function loadExecutionFiles(workspace, runId, jobId) {
     const files = await readExecutionFilesForRun(workspace, runId, jobId);
-    return summarizeSubjectFiles(files);
+    return summarizeCandidateFiles(files);
 }
 async function readExecutionFilesForRun(workspace, runId, jobId) {
     await assertExecutionJobInRun(workspace, runId, jobId);
@@ -430,7 +461,7 @@ function readLocalTraceSessions(job, role) {
         jobId: typeof session.jobId === "string" && session.jobId.length > 0
             ? session.jobId
             : job.id,
-        role: session.role === "optimizer" || session.role === "runner" || session.role === "engine"
+        role: session.role === "improver" || session.role === "runner" || session.role === "engine"
             ? session.role
             : role,
         kind: typeof session.kind === "string" && session.kind.length > 0 ? session.kind : "trace",
@@ -443,7 +474,7 @@ function readLocalTraceSessions(job, role) {
     }));
 }
 function traceSessionDisplayLabel(session, fallbackRole) {
-    const role = session.role === "optimizer" || session.role === "runner" || session.role === "engine"
+    const role = session.role === "improver" || session.role === "runner" || session.role === "engine"
         ? session.role
         : fallbackRole;
     return typeof session.label === "string" && session.label.length > 0
@@ -483,7 +514,16 @@ function readPreviewMode(params) {
     throw new LocalApiError("view must be diff, raw, or rendered.");
 }
 async function sendFile(response, filePath, contentType, method = "GET") {
-    const body = await fs.readFile(filePath);
+    let body;
+    try {
+        body = await fs.readFile(filePath);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            throw new LocalApiError("Workbench local asset not found.", 404);
+        }
+        throw error;
+    }
     response.writeHead(200, {
         "content-type": contentType,
         "content-length": body.byteLength,
@@ -504,7 +544,7 @@ async function sendFontFile(response, assetsRoot, url, method = "GET") {
     }
     await sendFile(response, path.join(assetsRoot, "fonts", fileName), "font/woff2", method);
 }
-async function sendHtml(response, method = "GET") {
+async function sendHtml(response, method = "GET", status = 200) {
     const body = `<!doctype html>
 <html lang="en">
   <head>
@@ -518,7 +558,7 @@ async function sendHtml(response, method = "GET") {
     <script type="module" src="/assets/client.js"></script>
   </body>
 </html>`;
-    response.writeHead(200, {
+    response.writeHead(status, {
         "content-type": "text/html; charset=utf-8",
         "content-length": Buffer.byteLength(body),
         "cache-control": "no-store",

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAsHA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA6BD,UAAU,iBAAiB;CAAG;AAqK9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CA8GjB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2IA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA4BD,UAAU,iBAAiB;CAAG;AAuK9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CAmHjB"}