@workbench-ai/workbench-core 0.0.46 → 0.0.47

package/dist/execution-traces.js

@@ -55,13 +55,14 @@ export function mergeWorkbenchExecutionTracesByJob(args) {
     const summaries = [];
     for (const job of args.jobs) {
         const prefix = sanitizeTraceComponent(job.id);
+        const traceJobId = job.jobId ?? job.id;
         spans.push(...job.trace.spans.map((span) => ({
             ...span,
             id: `${prefix}:${span.id}`,
             parent_id: span.parent_id ? `${prefix}:${span.parent_id}` : null,
             stage_id: args.stageId ?? span.stage_id,
             stage_run_index: null,
-            attributes: withTraceJobId(span.attributes, job.id),
+            attributes: withTraceJobId(span.attributes, traceJobId),
         })));
         events.push(...job.trace.events.map((event) => ({
             ...event,
@@ -69,7 +70,7 @@ export function mergeWorkbenchExecutionTracesByJob(args) {
             span_id: `${prefix}:${event.span_id}`,
             stage_id: args.stageId ?? event.stage_id,
             stage_run_index: null,
-            attributes: withTraceJobId(event.attributes, job.id),
+            attributes: withTraceJobId(event.attributes, traceJobId),
         })));
         summaries.push(...job.trace.summaries.map((summary) => ({
             ...summary,
@@ -84,6 +85,308 @@ export function mergeWorkbenchExecutionTracesByJob(args) {
         summaries: summaries.sort(compareTraceSummaries),
     };
 }
+export function buildWorkbenchTraceSessionsFromFiles(args) {
+    return args.files
+        .filter((file) => file.encoding === "utf8" && file.path.endsWith("/trace.json"))
+        .sort((left, right) => traceFileDisplayOrder(left.path) - traceFileDisplayOrder(right.path) ||
+        left.path.localeCompare(right.path))
+        .flatMap((file, index) => {
+        const trace = readWorkbenchExecutionTraceFile(file);
+        if (!trace) {
+            return [];
+        }
+        const prefix = traceFilePrefix(file.path, index);
+        const role = traceRoleForFilePath(file.path, args.purpose ?? null, args.fallbackRole);
+        return [{
+                id: `${args.job.id}:${prefix}`,
+                jobId: args.job.id,
+                role,
+                kind: traceSessionKindForFilePath(file.path, role),
+                label: traceSessionLabel(file.path, role),
+                sourcePath: file.path,
+                trace: prefixTraceFileIds(trace, prefix),
+                metadata: {
+                    trace_file: file.path,
+                },
+            }];
+    });
+}
+export function combineWorkbenchTraceSessions(sessions) {
+    return {
+        trace_id: sessions.length === 1 ? sessions[0].trace.trace_id : "combined-job-trace",
+        spans: sessions.flatMap((session) => session.trace.spans).sort(compareTraceSpans),
+        events: sessions.flatMap((session) => session.trace.events).sort(compareTraceEvents),
+        summaries: sessions.flatMap((session) => session.trace.summaries).sort(compareTraceSummaries),
+    };
+}
+export function readWorkbenchExecutionTraceFiles(files) {
+    const traces = files
+        .filter((file) => file.encoding === "utf8" && file.path.endsWith("/trace.json"))
+        .sort((left, right) => left.path.localeCompare(right.path))
+        .flatMap((file, index) => {
+        const trace = readWorkbenchExecutionTraceFile(file);
+        return trace ? [prefixTraceFileIds(trace, traceFilePrefix(file.path, index))] : [];
+    });
+    if (traces.length === 0) {
+        return null;
+    }
+    return {
+        trace_id: traces.length === 1 ? traces[0].trace_id : "combined-job-trace",
+        spans: traces.flatMap((trace) => trace.spans).sort(compareTraceSpans),
+        events: traces.flatMap((trace) => trace.events).sort(compareTraceEvents),
+        summaries: traces.flatMap((trace) => trace.summaries).sort(compareTraceSummaries),
+    };
+}
+export function traceSessionLabel(filePath, role) {
+    const innerPath = traceSessionInnerPath(filePath);
+    if (innerPath === "runner/session") {
+        return "Subject runner";
+    }
+    if (innerPath === "optimizer/session") {
+        return "Optimizer";
+    }
+    const parts = innerPath
+        .split("/")
+        .filter((part) => part.length > 0 && part !== "session");
+    const label = parts.slice(-2).join(" ") || role;
+    return formatTraceLabelText(label);
+}
+function traceSessionKindForFilePath(filePath, role) {
+    const innerPath = traceSessionInnerPath(filePath);
+    const parts = innerPath
+        .split("/")
+        .filter((part) => part.length > 0 && part !== "session");
+    return parts.at(-1) ?? role;
+}
+function readWorkbenchExecutionTraceFile(file) {
+    const traceRecord = parseJsonObject(file.content);
+    if (!traceRecord) {
+        return null;
+    }
+    const spans = Array.isArray(traceRecord.spans)
+        ? traceRecord.spans.map(readTraceSpan).filter((span) => span !== null)
+        : [];
+    const events = Array.isArray(traceRecord.events)
+        ? traceRecord.events.map(readTraceEvent).filter((event) => event !== null)
+        : [];
+    const summaries = Array.isArray(traceRecord.summaries)
+        ? traceRecord.summaries.map(readTraceSummary).filter((summary) => summary !== null)
+        : [];
+    if (spans.length === 0 && events.length === 0 && summaries.length === 0) {
+        return null;
+    }
+    return {
+        trace_id: readString(traceRecord.trace_id) ?? "agent-trace",
+        spans,
+        events,
+        summaries,
+    };
+}
+function prefixTraceFileIds(trace, prefix) {
+    return {
+        trace_id: `${prefix}:${trace.trace_id}`,
+        spans: trace.spans.map((span) => ({
+            ...span,
+            id: `${prefix}:${span.id}`,
+            parent_id: span.parent_id ? `${prefix}:${span.parent_id}` : null,
+            attributes: {
+                ...span.attributes,
+                trace_file: prefix,
+            },
+        })),
+        events: trace.events.map((event) => ({
+            ...event,
+            id: `${prefix}:${event.id}`,
+            span_id: `${prefix}:${event.span_id}`,
+            attributes: {
+                ...event.attributes,
+                trace_file: prefix,
+            },
+        })),
+        summaries: trace.summaries.map((summary) => ({ ...summary })),
+    };
+}
+function traceFilePrefix(filePath, index) {
+    const safe = filePath
+        .replace(/^\.workbench\/traces\//u, "")
+        .replace(/\/trace\.json$/u, "")
+        .replace(/[^A-Za-z0-9_-]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+    return safe || `trace-${index + 1}`;
+}
+function traceFileDisplayOrder(filePath) {
+    if (filePath.includes("/runner/") || filePath.includes("/optimizer/")) {
+        return 0;
+    }
+    return 1;
+}
+function traceRoleForFilePath(filePath, purpose, fallbackRole) {
+    if (filePath.includes("/runner/")) {
+        return "runner";
+    }
+    if (filePath.includes("/optimizer/") || purpose === "improve") {
+        return "optimizer";
+    }
+    if (filePath.includes("/engine/")) {
+        return "engine";
+    }
+    return fallbackRole;
+}
+function traceSessionInnerPath(filePath) {
+    const withoutTraceFile = filePath.replace(/\/trace\.json$/u, "");
+    const markerIndexes = [
+        withoutTraceFile.indexOf("/runner/"),
+        withoutTraceFile.indexOf("/optimizer/"),
+        withoutTraceFile.indexOf("/engine/"),
+    ].filter((index) => index >= 0);
+    const firstMarker = Math.min(...markerIndexes);
+    if (Number.isFinite(firstMarker)) {
+        return withoutTraceFile.slice(firstMarker + 1);
+    }
+    return withoutTraceFile.replace(/^\.workbench\/traces\/[^/]+\//u, "");
+}
+function formatTraceLabelText(value) {
+    return value
+        .replace(/[_-]+/g, " ")
+        .replace(/\s+/g, " ")
+        .trim()
+        .replace(/^\w/u, (match) => match.toUpperCase());
+}
+function readTraceSpan(value) {
+    const record = jsonRecord(value);
+    if (!record) {
+        return null;
+    }
+    const id = readString(record.id);
+    const kind = traceSpanKind(record.kind);
+    const status = traceStatus(record.status);
+    const startedAt = readString(record.started_at);
+    if (!id || !kind || !status || !startedAt) {
+        return null;
+    }
+    return {
+        id,
+        parent_id: readString(record.parent_id),
+        attempt_number: readPositiveInteger(record.attempt_number) ?? 1,
+        stage_id: readString(record.stage_id),
+        stage_run_index: readInteger(record.stage_run_index),
+        kind,
+        title: readString(record.title) ?? id,
+        status,
+        started_at: startedAt,
+        ended_at: readString(record.ended_at),
+        attributes: (jsonRecord(record.attributes) ?? {}),
+    };
+}
+function readTraceEvent(value) {
+    const record = jsonRecord(value);
+    if (!record) {
+        return null;
+    }
+    const id = readString(record.id);
+    const spanId = readString(record.span_id);
+    const kind = traceEventKind(record.kind);
+    const at = readString(record.at);
+    if (!id || !spanId || !kind || !at) {
+        return null;
+    }
+    return {
+        id,
+        span_id: spanId,
+        attempt_number: readPositiveInteger(record.attempt_number) ?? 1,
+        stage_id: readString(record.stage_id),
+        stage_run_index: readInteger(record.stage_run_index),
+        kind,
+        at,
+        message: readString(record.message) ?? kind,
+        attributes: (jsonRecord(record.attributes) ?? {}),
+    };
+}
+function readTraceSummary(value) {
+    const record = jsonRecord(value);
+    if (!record) {
+        return null;
+    }
+    const status = traceStatus(record.status);
+    const startedAt = readString(record.started_at);
+    if (!status || !startedAt) {
+        return null;
+    }
+    return {
+        attempt_number: readPositiveInteger(record.attempt_number) ?? 1,
+        stage_id: readString(record.stage_id),
+        stage_run_index: readInteger(record.stage_run_index),
+        status,
+        started_at: startedAt,
+        ended_at: readString(record.ended_at),
+        duration_ms: readNonNegativeInteger(record.duration_ms) ?? 0,
+        tool_call_count: readNonNegativeInteger(record.tool_call_count) ?? 0,
+        input_tokens: readNonNegativeInteger(record.input_tokens),
+        output_tokens: readNonNegativeInteger(record.output_tokens),
+        usage: jsonRecord(record.usage),
+        final_output_present: record.final_output_present === true,
+        error_message: readString(record.error_message),
+    };
+}
+function parseJsonObject(source) {
+    try {
+        return jsonRecord(JSON.parse(source));
+    }
+    catch {
+        return null;
+    }
+}
+function jsonRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function readString(value) {
+    return typeof value === "string" && value.length > 0 ? value : null;
+}
+function readInteger(value) {
+    return typeof value === "number" && Number.isInteger(value) ? value : null;
+}
+function readPositiveInteger(value) {
+    const integer = readInteger(value);
+    return integer !== null && integer > 0 ? integer : null;
+}
+function readNonNegativeInteger(value) {
+    const integer = readInteger(value);
+    return integer !== null && integer >= 0 ? integer : null;
+}
+function traceSpanKind(value) {
+    return value === "hook" ||
+        value === "stage" ||
+        value === "turn" ||
+        value === "tool_call" ||
+        value === "assistant_output" ||
+        value === "usage" ||
+        value === "gate" ||
+        value === "action" ||
+        value === "error"
+        ? value
+        : null;
+}
+function traceEventKind(value) {
+    return value === "status" ||
+        value === "message" ||
+        value === "output" ||
+        value === "usage" ||
+        value === "error" ||
+        value === "note"
+        ? value
+        : null;
+}
+function traceStatus(value) {
+    return value === "running" ||
+        value === "completed" ||
+        value === "failed" ||
+        value === "canceled" ||
+        value === "warning"
+        ? value
+        : null;
+}
 function withTraceJobId(attributes, jobId) {
     return {
         ...attributes,

package/dist/generic-spec.d.ts

@@ -1,4 +1,4 @@
-import type { EngineResolveBinding, SurfaceSnapshotFile, WorkbenchAdapterInvocation, WorkbenchExecutionNetworkPolicy, WorkbenchExecutionResources, WorkbenchSpecValidation } from "@workbench-ai/workbench-contract";
+import { type EngineResolveBinding, type SurfaceSnapshotFile, type WorkbenchAdapterInvocation, type WorkbenchExecutionNetworkPolicy, type WorkbenchExecutionResources, type WorkbenchSpecValidation } from "@workbench-ai/workbench-contract";
 import type { WorkbenchEngineCase, WorkbenchEngineCaseSpec } from "@workbench-ai/workbench-protocol";
 export declare const BENCHMARK_SPEC_FILE = "benchmark.yaml";
 export interface WorkbenchRuntimeSpec {
@@ -15,6 +15,9 @@ export interface WorkbenchRuntimeSpec {
 export interface WorkbenchPathRef {
     path: string;
 }
+export interface WorkbenchSubjectPrepareSpec {
+    command: string;
+}
 export interface AuthoredBenchmarkSpec {
     version: 3;
     name: string;
@@ -27,6 +30,7 @@ export interface WorkbenchSubjectManifestSpec {
     name: string;
     description?: string;
     files: WorkbenchPathRef;
+    prepare?: WorkbenchSubjectPrepareSpec;
     adapters: string[];
     run: WorkbenchAdapterInvocation;
 }
@@ -58,6 +62,7 @@ export interface GenericRunSpec {
         name: string;
         description?: string;
         files: WorkbenchPathRef;
+        prepare?: WorkbenchSubjectPrepareSpec;
     };
     optimizer?: {
         name: string;
@@ -105,8 +110,8 @@ export declare function engineCaseFilesForRuntimeInput(args: {
     spec: GenericRunSpec;
     engineCase: WorkbenchEngineCase;
 }): SurfaceSnapshotFile[];
-export declare function engineCaseSubjectVisibleFiles(engineCase: WorkbenchEngineCase): SurfaceSnapshotFile[];
-export declare function engineCaseEnginePrivateFiles(engineCase: WorkbenchEngineCase): SurfaceSnapshotFile[];
+export declare function engineCasePublicFiles(engineCase: WorkbenchEngineCase): SurfaceSnapshotFile[];
+export declare function engineCasePrivateFiles(engineCase: WorkbenchEngineCase): SurfaceSnapshotFile[];
 export declare function runtimeResources(runtime: WorkbenchRuntimeSpec): WorkbenchExecutionResources;
 export declare function runtimeNetwork(runtime: WorkbenchRuntimeSpec): WorkbenchExecutionNetworkPolicy;
 export declare function runtimeSandboxRef(runtime: WorkbenchRuntimeSpec): string;

package/dist/generic-spec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generic-spec.d.ts","sourceRoot":"","sources":["../src/generic-spec.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EAEpB,mBAAmB,EACnB,0BAA0B,EAC1B,+BAA+B,EAC/B,2BAA2B,EAC3B,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAG1C,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAEpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE;QACV,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,+BAA+B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,gBAAgB,CAAC;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAED,MAAM,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAE/D,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,0BAA0B,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,CAAC,CAAC;IACX,SAAS,EAAE,qBAAqB,CAAC;IACjC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,0BAA0B,CAAC;KACpC,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,gBAAgB,CAAC;KACzB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,WAAW,EAAE,oBAAoB,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;IACnC,aAAa,EAAE,0BAA0B,CAAC;IAC1C,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,GAAG,EAAE,0BAA0B,CAAC;IAChC,SAAS,EAAE,0BAA0B,CAAC;CACvC;AAED,MAAM,MAAM,qBAAqB,GAAG,uBAAuB,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,oBAAoB,CAAC;IAClC,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAQD,eAAO,MAAM,2BAA2B,EAAE,2BAKzC,CAAC;AAEF,wBAAgB,mCAAmC,CACjD,MAAM,EAAE,MAAM,GACb,uBAAuB,CAmBzB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,MAAM,GACb,cAAc,CAsChB;AAED,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,MAAM,GACb,oBAAoB,CAEtB;AAED,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,cAAc,GACnB,oBAAoB,CAStB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,cAAc,CAMjB;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,uBAAuB,CA4B1B;AAED,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,uBAAuB,GAC9B,MAAM,CAER;AAED,wBAAgB,8BAA8B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAIxE;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE;IACrD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,qBAAqB,CAAC;CACnC,GAAG,iCAAiC,CAMpC;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,cAAc,GAAG,0BAA0B,CAE/F;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,mBAAmB,CAAC;CACjC,GAAG,mBAAmB,EAAE,CAGxB;AAED,wBAAgB,6BAA6B,CAC3C,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,4BAA4B,CAC1C,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,GAC5B,2BAA2B,CAiB7B;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,oBAAoB,GAC5B,+BAA+B,CAEjC;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAEvE"}
+{"version":3,"file":"generic-spec.d.ts","sourceRoot":"","sources":["../src/generic-spec.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,+BAA+B,EACpC,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC7B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAG1C,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAEpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE;QACV,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,+BAA+B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,gBAAgB,CAAC;IACxB,OAAO,CAAC,EAAE,2BAA2B,CAAC;IACtC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAED,MAAM,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAE/D,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,0BAA0B,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,CAAC,CAAC;IACX,SAAS,EAAE,qBAAqB,CAAC;IACjC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,0BAA0B,CAAC;KACpC,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,gBAAgB,CAAC;QACxB,OAAO,CAAC,EAAE,2BAA2B,CAAC;KACvC,CAAC;IACF,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,WAAW,EAAE,oBAAoB,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;IACnC,aAAa,EAAE,0BAA0B,CAAC;IAC1C,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,GAAG,EAAE,0BAA0B,CAAC;IAChC,SAAS,EAAE,0BAA0B,CAAC;CACvC;AAED,MAAM,MAAM,qBAAqB,GAAG,uBAAuB,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,oBAAoB,CAAC;IAClC,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAQD,eAAO,MAAM,2BAA2B,EAAE,2BAKzC,CAAC;AAEF,wBAAgB,mCAAmC,CACjD,MAAM,EAAE,MAAM,GACb,uBAAuB,CAmBzB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,MAAM,GACb,cAAc,CAsChB;AAED,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,MAAM,GACb,oBAAoB,CAEtB;AAED,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,cAAc,GACnB,oBAAoB,CAStB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,cAAc,CAMjB;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,uBAAuB,CA4B1B;AAED,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,uBAAuB,GAC9B,MAAM,CAER;AAED,wBAAgB,8BAA8B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAIxE;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE;IACrD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,qBAAqB,CAAC;CACnC,GAAG,iCAAiC,CAMpC;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,cAAc,GAAG,0BAA0B,CAE/F;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,mBAAmB,CAAC;CACjC,GAAG,mBAAmB,EAAE,CAGxB;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,GAC5B,2BAA2B,CAiB7B;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,oBAAoB,GAC5B,+BAA+B,CAEjC;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAEvE"}

package/dist/generic-spec.js

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { isWorkbenchExecutionNetworkEgress, } from "@workbench-ai/workbench-contract";
 import YAML from "yaml";
 export const BENCHMARK_SPEC_FILE = "benchmark.yaml";
 export const DEFAULT_EXECUTION_RESOURCES = {
@@ -110,15 +111,15 @@ export function engineResolveInvocationForSpec(spec) {
 }
 export function engineCaseFilesForRuntimeInput(args) {
     void args.spec;
-    return engineCaseSubjectVisibleFiles(args.engineCase);
+    return engineCasePublicFiles(args.engineCase);
 }
-export function engineCaseSubjectVisibleFiles(engineCase) {
-    return (engineCase.files.subjectVisible ?? [])
+export function engineCasePublicFiles(engineCase) {
+    return (engineCase.files.public ?? [])
         .map((file) => ({ ...file }))
         .sort((left, right) => left.path.localeCompare(right.path));
 }
-export function engineCaseEnginePrivateFiles(engineCase) {
-    return (engineCase.files.enginePrivate ?? [])
+export function engineCasePrivateFiles(engineCase) {
+    return (engineCase.files.private ?? [])
         .map((file) => ({ ...file }))
         .sort((left, right) => left.path.localeCompare(right.path));
 }
@@ -154,6 +155,7 @@ function genericSpecFromAuthoredBundle(source) {
             name: source.subject.name,
             ...(source.subject.description ? { description: source.subject.description } : {}),
             files: cloneJson(source.subject.files),
+            ...(source.subject.prepare ? { prepare: cloneJson(source.subject.prepare) } : {}),
         },
         ...(source.optimizer
             ? {
@@ -229,6 +231,7 @@ function normalizeSubjectRecord(record, label, errors) {
         "name",
         "description",
         "files",
+        "prepare",
         "adapters",
         "run",
     ], errors);
@@ -236,6 +239,7 @@ function normalizeSubjectRecord(record, label, errors) {
     const name = readRequiredString(record.name, `${label}.name`, errors);
     const description = readOptionalString(record.description, `${label}.description`, errors);
     const files = normalizePathRef(record.files, `${label}.files`, errors);
+    const prepare = normalizeSubjectPrepare(record.prepare, `${label}.prepare`, errors);
     const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
     const run = normalizePhaseAdapter(record.run, `${label}.run`, errors);
     return name && files && run
@@ -244,11 +248,24 @@ function normalizeSubjectRecord(record, label, errors) {
             name,
             ...(description ? { description } : {}),
             files,
+            ...(prepare ? { prepare } : {}),
             adapters,
             run,
         }
         : null;
 }
+function normalizeSubjectPrepare(value, label, errors) {
+    if (value === undefined) {
+        return undefined;
+    }
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return undefined;
+    }
+    rejectUnknownKeys(record, label, ["command"], errors);
+    const command = readRequiredString(record.command, `${label}.command`, errors);
+    return command ? { command } : undefined;
+}
 function normalizeOptimizerRecord(record, label, errors) {
     if (!record) {
         return null;
@@ -438,41 +455,13 @@ function normalizeAdapterSources(value, label, errors) {
     return [...new Set(sources)];
 }
 function normalizeNetworkConfig(network, label, errors) {
-    rejectUnknownKeys(network, label, ["egress", "allow"], errors);
+    rejectUnknownKeys(network, label, ["egress"], errors);
     const egress = readOptionalString(network.egress, `${label}.egress`, errors) ?? "open";
-    if (egress !== "none" && egress !== "open" && egress !== "allowlist") {
-        errors.push(`${label}.egress must be none, open, or allowlist.`);
+    if (!isWorkbenchExecutionNetworkEgress(egress)) {
+        errors.push(`${label}.egress must be none or open.`);
         return null;
     }
-    const allow = network.allow === undefined
-        ? undefined
-        : normalizeNetworkAllowList(network.allow, `${label}.allow`, errors);
-    if (egress !== "allowlist") {
-        if (network.allow !== undefined) {
-            errors.push(`${label}.allow is only supported when ${label}.egress is allowlist.`);
-        }
-        return { egress };
-    }
-    if (!allow || allow.length === 0) {
-        errors.push(`${label}.allow must contain at least one host when ${label}.egress is allowlist.`);
-    }
-    return {
-        egress,
-        ...(allow && allow.length > 0 ? { allow } : {}),
-    };
-}
-function normalizeNetworkAllowList(value, label, errors) {
-    if (!Array.isArray(value)) {
-        errors.push(`${label} must be an array of hosts.`);
-        return [];
-    }
-    return value.flatMap((entry, index) => {
-        if (typeof entry !== "string" || entry.trim().length === 0) {
-            errors.push(`${label}[${index}] must be a non-empty string.`);
-            return [];
-        }
-        return [entry.trim()];
-    });
+    return { egress };
 }
 function normalizePhaseAdapter(value, label, errors) {
     const spec = readAdapterRecord(value, label, errors);

package/dist/index.d.ts

@@ -1,16 +1,18 @@
-import type { AuthoredWorkbenchSourceDocument, SubjectCasePhaseRef, SubjectCaseReview, SubjectFilePreview, SubjectFileSummary, SubjectLineageGraph, SubjectRecord, SubjectSummary, EvalCaseResult, EvaluationResultRecord, HostedWorkbenchEnvironment, HostedWorkbenchEnvironmentVersion, HostedWorkbenchFileInput, HostedWorkbenchJob, Json, RuntimeEvent, SurfaceSnapshotFile, UsageSummary, WorkbenchSubjectPatch, WorkbenchExecutionSpec, WorkbenchResult } from "@workbench-ai/workbench-contract";
+import type { AuthoredWorkbenchSourceDocument, SubjectCaseExecutionRef, SubjectCaseReview, SubjectFilePreview, SubjectFileSummary, SubjectLineageGraph, SubjectRecord, SubjectSummary, EvalCaseResult, EvaluationScorecard, HostedWorkbenchEnvironment, HostedWorkbenchEnvironmentVersion, HostedWorkbenchFileInput, HostedWorkbenchJob, Json, RuntimeEvent, SurfaceSnapshotFile, UsageSummary, WorkbenchSubjectPatch, WorkbenchExecutionCapability, WorkbenchExecutionSpec, WorkbenchResult } from "@workbench-ai/workbench-contract";
+import { type WorkbenchAdapterOperationExecutor, type WorkbenchAdapterOperationResult } from "@workbench-ai/workbench-protocol";
 import { type GenericEngineCaseSpec, type GenericRunSpec, type WorkbenchEngineCase } from "./generic-spec.ts";
 import type { WorkbenchExecutionRuntimeInput } from "./execution-runtime-types.ts";
 import { createWorkbenchExecutionCapability, type SandboxExecutionFileStore, type SandboxPlane } from "./sandbox-plane.ts";
-export { BENCHMARK_SPEC_FILE, DEFAULT_EXECUTION_RESOURCES, engineCaseEnginePrivateFiles, engineCaseFilesForRuntimeInput, engineCaseSubjectVisibleFiles, engineResolveInvocationForSpec, engineResolveBindingForSpec, engineResolveBindingForSourceYaml, isWorkbenchSubjectManifestPath, parseWorkbenchSourceFiles, resolveEngineCaseExecutionConfig, resolveWorkbenchResolvedSourceYaml, resolveWorkbenchSourceFiles, serializeWorkbenchResolvedSourceYaml, validateWorkbenchResolvedSourceYaml, type AuthoredBenchmarkSpec, type AuthoredOptimizerSpec, type GenericRunSpec, type GenericEngineCaseSpec, type ResolvedSubjectSpec, type WorkbenchEngineCase, type WorkbenchResolvedSource, type WorkbenchSubjectManifestSpec, } from "./generic-spec.ts";
-export { adapterCommandName, cloneWorkbenchAdapterManifest, collectWorkbenchAdapterAuthRequirements, collectWorkbenchAdapterInvocations, parseWorkbenchAdapterManifest, workbenchAdapterManifestRequiresAuth, workbenchAdapterManifestSupportsOperation, workbenchAdapterOperationCommand, withDefaultWorkbenchAdapterAuth, withDefaultWorkbenchAdapterAuthProfiles, type WorkbenchPrimitiveAdapterOperation, type WorkbenchAdapterOperation, type WorkbenchAdapterOperationManifest, type WorkbenchAdapterSlotManifest, type WorkbenchAdapterAuthRequirement, type WorkbenchAdapterAuthManifest, type WorkbenchAdapterAuthMethodManifest, type WorkbenchAdapterInvocationLike, type WorkbenchAdapterManifest, } from "@workbench-ai/workbench-protocol";
+export { BENCHMARK_SPEC_FILE, DEFAULT_EXECUTION_RESOURCES, engineCasePrivateFiles, engineCaseFilesForRuntimeInput, engineCasePublicFiles, engineResolveInvocationForSpec, engineResolveBindingForSpec, engineResolveBindingForSourceYaml, isWorkbenchSubjectManifestPath, parseWorkbenchSourceFiles, resolveEngineCaseExecutionConfig, resolveWorkbenchResolvedSourceYaml, resolveWorkbenchSourceFiles, runtimeNetwork, runtimeResources, serializeWorkbenchResolvedSourceYaml, validateWorkbenchResolvedSourceYaml, type AuthoredBenchmarkSpec, type AuthoredOptimizerSpec, type GenericRunSpec, type GenericEngineCaseSpec, type ResolvedSubjectSpec, type WorkbenchEngineCase, type WorkbenchResolvedSource, type WorkbenchSubjectManifestSpec, } from "./generic-spec.ts";
+export { composeRuntimeDockerfileWithAdapterInstallers, type WorkbenchRuntimeAdapterInstaller, type WorkbenchRuntimeAdapterInstallerFile, } from "./runtime-dockerfile.ts";
+export { adapterCommandName, cloneWorkbenchAdapterManifest, collectWorkbenchAdapterAuthRequirements, collectWorkbenchAdapterInvocations, parseWorkbenchAdapterManifest, workbenchAdapterManifestRequiresAuth, workbenchAdapterManifestSupportsOperation, workbenchAdapterOperationCommand, workbenchAdapterOperationExecutor, withDefaultWorkbenchAdapterAuth, withDefaultWorkbenchAdapterAuthProfiles, type WorkbenchPrimitiveAdapterOperation, type WorkbenchAdapterOperation, type WorkbenchAdapterOperationExecutor, type WorkbenchAdapterOperationManifest, type WorkbenchAdapterSlotManifest, type WorkbenchAdapterAuthRequirement, type WorkbenchAdapterAuthManifest, type WorkbenchAdapterAuthMethodManifest, type WorkbenchAdapterInvocationLike, type WorkbenchAdapterManifest, } from "@workbench-ai/workbench-protocol";
 export { adapterAuthEnv, createWorkbenchAdapterAuthBundle, defaultWorkbenchAdapterAuthStoreRoot, localWorkbenchAdapterAuthStore, normalizeWorkbenchAdapterAuthTarget, parseWorkbenchAdapterAuthTarget, sanitizeWorkbenchAdapterAuthBundle, type WorkbenchAdapterAuthBundle, type WorkbenchAdapterAuthEnvVar, type WorkbenchAdapterAuthFile, type WorkbenchAdapterAuthStatus, type WorkbenchAdapterAuthStatusRecord, type WorkbenchAdapterAuthStore, type WorkbenchAdapterAuthTarget, } from "./adapter-auth.ts";
-export type { WorkbenchExecutionRuntimeInput, WorkbenchWorkloadPhaseCommand, } from "./execution-runtime-types.ts";
+export type { WorkbenchExecutionRuntimeInput, WorkbenchWorkloadStepCommand, } from "./execution-runtime-types.ts";
 export { asRuntimeRecord, importNodeModule, nodeBuiltin, normalizeWorkbenchWorkerId, normalizeRuntimeRegistry, quoteShellArg, resolveDockerRuntimeImageRef, resolveWorkbenchWorkerId, } from "./runtime-utils.ts";
 export { assignUsageRole, extractExecutionUsageFromTrace, mergeUsageSummaries, } from "./execution-usage.ts";
 export { createWorkbenchProgressStdoutParser, publishWorkbenchProgressStdoutEnvelope, } from "./execution-events.ts";
 export { resolveSandboxTemplateImage, } from "./sandbox-backends/template-images.ts";
-export { readOutputTraceFiles, workbenchTracePhaseDirectory, workbenchTraceRunDirectory, workbenchTraceRunDirectoryName, } from "./trace-files.ts";
+export { readOutputTraceFiles, workbenchTraceExecutionDirectory, workbenchTraceRunDirectory, workbenchTraceRunDirectoryName, } from "./trace-files.ts";
 export { assertWorkbenchAdapterOperationSupport, assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterOperationIssues, collectWorkbenchAdapterOperationRequirements, ensureWorkbenchAdapterOutputDir, WORKBENCH_ADAPTER_RESULT_FILE, normalizeWorkbenchAdapterOperationRequest, normalizeWorkbenchAdapterOperationResult, readWorkbenchAdapterOperationRequest, readWorkbenchAdapterOperationResult, workbenchAdapterOperationResultPath, writeWorkbenchAdapterOperationResult, type WorkbenchAdapterOperationRequest, type WorkbenchAdapterOperationResult, type WorkbenchAdapterOperationResultValue, type WorkbenchAdapterOperationRequirement, type WorkbenchEngineResolveResult, type WorkbenchEngineCaseSpec, } from "@workbench-ai/workbench-protocol";
 export { applyWorkbenchSubjectPatch, type ApplyWorkbenchSubjectPatchInput, } from "./subject-patch.ts";
 export { createWorkbenchSandboxFileStore, createSandboxAdapterRequest, executionResultFromCompletedSandboxJob, materializeWorkbenchSandboxInput, readWorkbenchExecutionSpec, sanitizeWorkbenchExecutionJobForSandbox, } from "./sandbox-inputs.ts";
@@ -19,15 +21,15 @@ export { createBaselineSubjectExecution, createBaselineSubjectJob, createWorkben
 export { addCapacity, capacityFits, runWorkbenchExecutionDag, subtractCapacity, workbenchJobDependencies, workbenchJobHostCost, workbenchJobResources, type WorkbenchExecutionDagCapacity, type WorkbenchExecutionDagResult, type WorkbenchExecutionDagRunInput, } from "./execution-scheduler.ts";
 export { assertWorkbenchExecutionIsolation, collectWorkbenchExecutionIsolationIssues, validateWorkbenchExecutionOutputPayloads, type WorkbenchExecutionOutputPayloads, } from "./execution-outputs.ts";
 export { collectSandboxAllocationScopeIssues, collectExecutionCapabilityScopeIssues, collectSandboxHandleScopeIssues, createWorkbenchSandboxAllocation, createWorkbenchSandboxExecutionMetadata, createWorkbenchExecutionCapability, executeValidatedSandboxExecution, type SandboxExecutionFileStore, type SandboxExecutionOptions, type SandboxBackendCapabilities, type SandboxBackendDescriptor, type SandboxCreateRequest, type SandboxEnvironmentImage, type SandboxExecRequest, type SandboxHandle, type SandboxMaterializedInput, type SandboxPlane, type ValidatedSandboxExecutionResult, } from "./sandbox-plane.ts";
-export { buildSubjectCasePhaseRefs, buildWorkbenchTracePhases, isWorkbenchPhaseActive, readWorkbenchExecutionId, readWorkbenchExecutionMetadataNumber, readWorkbenchExecutionMetadataString, readWorkbenchExecutionPurpose, resolveWorkbenchJobGroupStatus, } from "./execution-phases.ts";
-export { finalizeWorkbenchExecutionTraceForJob, mergeWorkbenchExecutionTracesByJob, type WorkbenchTraceMergeJob, } from "./execution-traces.ts";
+export { buildSubjectCaseExecutionRefs, buildWorkbenchExecutionEvidence, isWorkbenchExecutionActive, readWorkbenchExecutionId, readWorkbenchExecutionMetadataNumber, readWorkbenchExecutionMetadataString, readWorkbenchExecutionPurpose, resolveWorkbenchJobGroupStatus, } from "./execution-evidence.ts";
+export { buildWorkbenchTraceSessionsFromFiles, combineWorkbenchTraceSessions, finalizeWorkbenchExecutionTraceForJob, mergeWorkbenchExecutionTracesByJob, readWorkbenchExecutionTraceFiles, traceSessionLabel, type WorkbenchTraceMergeJob, } from "./execution-traces.ts";
 export { DOCKER_SANDBOX_BACKEND, assertSandboxHostHealthForProvider, createDockerSandboxBackendDescriptor, createDockerSandboxPlane, resolveWorkbenchSandboxProviderName, sandboxProviderAdmissionForResources, sandboxProviderDefaultMaxConcurrentJobs, sandboxProviderLeaseScope, sandboxHostHealthExpectationForProvider, type SandboxProviderAdmission, type SandboxProviderHostCost, type SandboxProviderLeaseRequest, type SandboxProviderRequestedResources, type SandboxHostHealthExpectation, type WorkbenchSandboxProviderName, } from "./sandbox-backends/index.ts";
 export type { WorkbenchExecutionEventPublisher, WorkbenchExecutionProgressTarget, } from "./execution-events.ts";
-export type { SubjectCaseReview, SubjectRecord, EngineResolveBinding, EvaluationResultRecord, HostedWorkbenchJob, Json, RunSummary, RuntimeEvent, SurfaceSnapshotFile, WorkbenchExecutionCapability, WorkbenchExecutionTrace, WorkbenchSandboxHandle, WorkbenchSandboxExecutionMetadata, } from "@workbench-ai/workbench-contract";
+export type { SubjectCaseReview, SubjectRecord, EngineResolveBinding, EvaluationScorecard, HostedWorkbenchJob, Json, RunSummary, RuntimeEvent, SurfaceSnapshotFile, WorkbenchExecutionCapability, WorkbenchExecutionTrace, WorkbenchTraceSession, WorkbenchSandboxHandle, WorkbenchSandboxExecutionMetadata, } from "@workbench-ai/workbench-contract";
 export interface WorkbenchRunMaterialization {
     subjects: SubjectRecord[];
     subjectFiles: Record<string, SurfaceSnapshotFile[]>;
-    evaluations: EvaluationResultRecord[];
+    evaluations: EvaluationScorecard[];
     activeSubjectId: string | null;
     selectedSubject: SubjectRecord | null;
     completedJobCount: number;
@@ -52,6 +54,8 @@ export interface WorkbenchRunWorkload {
 export interface RuntimeWorkloadResult {
     files: SurfaceSnapshotFile[];
     fileChanges: string[];
+    operationResults?: WorkbenchAdapterOperationResult[];
+    workspaceFiles?: SurfaceSnapshotFile[];
     subjectPatch?: WorkbenchSubjectPatch;
     result?: WorkbenchResult;
     metrics?: Record<string, number>;
@@ -86,6 +90,7 @@ export declare function materializeWorkbenchRunResult(args: {
     previousSubject?: SubjectRecord | null;
     existingSubjectCount: number;
 }): WorkbenchRunMaterialization;
+export declare function evaluationScorecardId(runId: string, subjectId: string): string;
 export declare function selectExecutionOutputFilesForInspection(args: {
     purpose: string | null | undefined;
     files: readonly SurfaceSnapshotFile[];
@@ -97,6 +102,10 @@ export declare function createSubjectRevisionTraceInputFiles(args: {
     jobs: readonly HostedWorkbenchJob[];
     events: readonly RuntimeEvent[];
 }): SurfaceSnapshotFile[];
+export declare function createSubjectEvaluationTraceInputFiles(args: {
+    subject?: SubjectRecord | null;
+    path?: string;
+}): SurfaceSnapshotFile[];
 export interface WorkbenchProjectSourceFilesInput {
     specSource?: string;
     specFiles?: readonly SurfaceSnapshotFile[];
@@ -134,7 +143,7 @@ export declare function createSubjectFilePreview(args: {
 export declare function createCaseReview(args: {
     subject: SubjectRecord;
     caseId: string;
-    phases?: SubjectCasePhaseRef[];
+    executions?: SubjectCaseExecutionRef[];
 }): SubjectCaseReview;
 export declare function createWorkbenchRunWorkload(args: {
     job: HostedWorkbenchJob;
@@ -150,8 +159,10 @@ export interface WorkbenchExecutionJobOptions {
     createSandboxPlaneForProvider?: (provider: string, args: WorkbenchExecutionRuntimeInput, startedAt: string, fileStore: SandboxExecutionFileStore) => SandboxPlane;
 }
 export declare function executeWorkbenchExecutionJob(args: WorkbenchExecutionRuntimeInput, options: WorkbenchExecutionJobOptions): Promise<HostedWorkbenchJob>;
+export declare function workbenchExecutionExecutorForRuntimeInput(args: Pick<WorkbenchExecutionRuntimeInput, "job" | "adapterManifests" | "runtimeControlOperation">): WorkbenchAdapterOperationExecutor;
 export declare function workbenchExecutionPurpose(job: HostedWorkbenchJob): WorkbenchExecutionSpec["purpose"] | null;
-export declare function executeAdapterInCurrentSandboxRuntime(args: WorkbenchExecutionRuntimeInput, execution: WorkbenchExecutionSpec, startedAt: string, capability: ReturnType<typeof createWorkbenchExecutionCapability>): Promise<HostedWorkbenchJob>;
+export declare function executeAdapterInCurrentRuntime(args: WorkbenchExecutionRuntimeInput, execution: WorkbenchExecutionSpec, startedAt: string, capability: ReturnType<typeof createWorkbenchExecutionCapability>): Promise<HostedWorkbenchJob>;
+export declare function executeRuntimeControlOperationSequenceInCurrentRuntime(args: WorkbenchExecutionRuntimeInput, execution: WorkbenchExecutionSpec, startedAt: string, capability?: WorkbenchExecutionCapability): Promise<HostedWorkbenchJob>;
 export declare function stageWorkbenchRunWorkload(root: string, workload: WorkbenchRunWorkload): Promise<void>;
 export declare function workloadTimeoutMs(spec: GenericRunSpec): number;
 export declare function findEnvironmentVersionForImage(image: string, versions: readonly HostedWorkbenchEnvironmentVersion[]): HostedWorkbenchEnvironmentVersion | null;