@workbench-ai/workbench-core 0.0.46 → 0.0.48

package/dist/execution-events.d.ts

@@ -29,8 +29,8 @@ export interface WorkbenchExecutionEventPublisher {
 }
 export declare const WORKBENCH_PROGRESS_STDOUT_PREFIX = "__WORKBENCH_PROGRESS__";
 export declare function createWorkbenchExecutionEventPublisher(context: WorkbenchExecutionEventPublisherContext): WorkbenchExecutionEventPublisher;
-export declare function publishCommandPhaseEvent(publisher: WorkbenchExecutionEventPublisher | undefined, args: {
-    phase: string;
+export declare function publishCommandStepEvent(publisher: WorkbenchExecutionEventPublisher | undefined, args: {
+    step: string;
     status: "started" | "succeeded" | "failed";
     role?: WorkbenchExecutionEvent["role"];
     exitCode?: number;

package/dist/execution-events.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execution-events.d.ts","sourceRoot":"","sources":["../src/execution-events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EACJ,uBAAuB,EACvB,4BAA4B,EAC5B,6BAA6B,EAC9B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,gCAAgC;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,uCAAuC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,gCAAgC,CAAC;CAC3C;AAED,MAAM,WAAW,4BAA4B;IAC3C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,6BAA6B,CAAC;IACtC,IAAI,CAAC,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAC1C,OAAO,EAAE,IAAI,CAAC;CACf;AAED,MAAM,WAAW,gCAAgC;IAC/C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,MAAM,EAAE,SAAS,4BAA4B,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,eAAO,MAAM,gCAAgC,2BAA2B,CAAC;AAEzE,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,uCAAuC,GAC/C,gCAAgC,CAsClC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,gCAAgC,GAAG,SAAS,EACvD,IAAI,EAAE;IACJ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC3C,IAAI,CAAC,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACA,OAAO,CAAC,IAAI,CAAC,CAef;AAyCD,MAAM,WAAW,+BAA+B;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE;QACJ,IAAI,EAAE,wBAAwB,CAAC;QAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,4BAA4B,CAAC;KACrC,CAAC;CACH;AAED,wBAAgB,mCAAmC,CACjD,UAAU,EAAE,CAAC,QAAQ,EAAE,+BAA+B,KAAK,IAAI,GAC9D;IACD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,KAAK,IAAI,IAAI,CAAC;CACf,CA0DA;AAED,wBAAsB,sCAAsC,CAC1D,QAAQ,EAAE,+BAA+B,GACxC,OAAO,CAAC,IAAI,CAAC,CAEf"}
+{"version":3,"file":"execution-events.d.ts","sourceRoot":"","sources":["../src/execution-events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,EACJ,uBAAuB,EACvB,4BAA4B,EAC5B,6BAA6B,EAC9B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,gCAAgC;IAC/C,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,uCAAuC;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,gCAAgC,CAAC;CAC3C;AAED,MAAM,WAAW,4BAA4B;IAC3C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,6BAA6B,CAAC;IACtC,IAAI,CAAC,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,EAAE,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAC1C,OAAO,EAAE,IAAI,CAAC;CACf;AAED,MAAM,WAAW,gCAAgC;IAC/C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,OAAO,CAAC,MAAM,EAAE,SAAS,4BAA4B,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,eAAO,MAAM,gCAAgC,2BAA2B,CAAC;AAEzE,wBAAgB,sCAAsC,CACpD,OAAO,EAAE,uCAAuC,GAC/C,gCAAgC,CAsClC;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,gCAAgC,GAAG,SAAS,EACvD,IAAI,EAAE;IACJ,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC3C,IAAI,CAAC,EAAE,uBAAuB,CAAC,MAAM,CAAC,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GACA,OAAO,CAAC,IAAI,CAAC,CAef;AAyCD,MAAM,WAAW,+BAA+B;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE;QACJ,IAAI,EAAE,wBAAwB,CAAC;QAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,EAAE,4BAA4B,CAAC;KACrC,CAAC;CACH;AAED,wBAAgB,mCAAmC,CACjD,UAAU,EAAE,CAAC,QAAQ,EAAE,+BAA+B,KAAK,IAAI,GAC9D;IACD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IACpC,KAAK,IAAI,IAAI,CAAC;CACf,CA0DA;AAED,wBAAsB,sCAAsC,CAC1D,QAAQ,EAAE,+BAA+B,GACxC,OAAO,CAAC,IAAI,CAAC,CAEf"}

package/dist/execution-events.js

@@ -38,16 +38,16 @@ export function createWorkbenchExecutionEventPublisher(context) {
         },
     };
 }
-export async function publishCommandPhaseEvent(publisher, args) {
+export async function publishCommandStepEvent(publisher, args) {
     if (!publisher?.enabled) {
         return;
     }
     await publisher.publish([{
             source: "command",
             ...(args.role ? { role: args.role } : {}),
-            schema: "workbench.execution.phase.v1",
+            schema: "workbench.execution.step.v1",
             payload: {
-                phase: args.phase,
+                step: args.step,
                 status: args.status,
                 ...(typeof args.exitCode === "number" ? { exitCode: args.exitCode } : {}),
                 ...(args.error ? { error: args.error } : {}),

package/dist/{execution-phases.d.ts → execution-evidence.d.ts}

@@ -1,21 +1,22 @@
-import type { SubjectCaseReview, HostedWorkbenchJob, HostedWorkbenchJobStatus, WorkbenchExecutionEventRole, WorkbenchExecutionSpec, WorkbenchExecutionTrace, WorkbenchTracePhase } from "@workbench-ai/workbench-contract";
-export declare function buildSubjectCasePhaseRefs(args: {
+import type { SubjectCaseReview, HostedWorkbenchJob, HostedWorkbenchJobStatus, WorkbenchExecutionEventRole, WorkbenchExecutionSpec, WorkbenchExecutionTrace, WorkbenchExecutionEvidence, WorkbenchTraceSession } from "@workbench-ai/workbench-contract";
+export declare function buildSubjectCaseExecutionRefs(args: {
     jobs: readonly HostedWorkbenchJob[];
     subjectId: string;
     caseId: string;
     sampleIndex?: number;
-}): SubjectCaseReview["phases"];
-export declare function buildWorkbenchTracePhases(args: {
+}): SubjectCaseReview["executions"];
+export declare function buildWorkbenchExecutionEvidence(args: {
     jobs: readonly HostedWorkbenchJob[];
     traceIdPrefix: string;
     traceForJob: (job: HostedWorkbenchJob, role: WorkbenchExecutionEventRole) => WorkbenchExecutionTrace;
-}): WorkbenchTracePhase[];
+    traceSessionsForJob?: (job: HostedWorkbenchJob, role: WorkbenchExecutionEventRole) => WorkbenchTraceSession[];
+}): WorkbenchExecutionEvidence[];
 export declare function readWorkbenchExecutionPurpose(job: HostedWorkbenchJob): WorkbenchExecutionSpec["purpose"] | null;
 export declare function readWorkbenchExecutionId(job: HostedWorkbenchJob): string | null;
 export declare function readWorkbenchExecutionMetadataString(job: HostedWorkbenchJob, key: string): string | null;
 export declare function readWorkbenchExecutionMetadataNumber(job: HostedWorkbenchJob, key: string): number | null;
-export declare function isWorkbenchPhaseActive(phase: SubjectCaseReview["phases"][number]): boolean;
+export declare function isWorkbenchExecutionActive(execution: SubjectCaseReview["executions"][number]): boolean;
 export declare function resolveWorkbenchJobGroupStatus(jobs: readonly {
     status: HostedWorkbenchJobStatus;
 }[]): HostedWorkbenchJobStatus;
-//# sourceMappingURL=execution-phases.d.ts.map
+//# sourceMappingURL=execution-evidence.d.ts.map

package/dist/execution-evidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-evidence.d.ts","sourceRoot":"","sources":["../src/execution-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EAExB,2BAA2B,EAC3B,sBAAsB,EACtB,uBAAuB,EACvB,0BAA0B,EAC1B,qBAAqB,EACtB,MAAM,kCAAkC,CAAC;AAI1C,wBAAgB,6BAA6B,CAAC,IAAI,EAAE;IAClD,IAAI,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAgElC;AAED,wBAAgB,+BAA+B,CAAC,IAAI,EAAE;IACpD,IAAI,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACpC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,CACX,GAAG,EAAE,kBAAkB,EACvB,IAAI,EAAE,2BAA2B,KAC9B,uBAAuB,CAAC;IAC7B,mBAAmB,CAAC,EAAE,CACpB,GAAG,EAAE,kBAAkB,EACvB,IAAI,EAAE,2BAA2B,KAC9B,qBAAqB,EAAE,CAAC;CAC9B,GAAG,0BAA0B,EAAE,CAgF/B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,sBAAsB,CAAC,SAAS,CAAC,GAAG,IAAI,CAQ1C;AAED,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,kBAAkB,GAAG,MAAM,GAAG,IAAI,CAG/E;AAED,wBAAgB,oCAAoC,CAClD,GAAG,EAAE,kBAAkB,EACvB,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,IAAI,CAGf;AAED,wBAAgB,oCAAoC,CAClD,GAAG,EAAE,kBAAkB,EACvB,GAAG,EAAE,MAAM,GACV,MAAM,GAAG,IAAI,CAGf;AAED,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,iBAAiB,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,GACjD,OAAO,CAET;AAED,wBAAgB,8BAA8B,CAC5C,IAAI,EAAE,SAAS;IAAE,MAAM,EAAE,wBAAwB,CAAA;CAAE,EAAE,GACpD,wBAAwB,CAc1B"}

package/dist/{execution-phases.js → execution-evidence.js}

@@ -1,32 +1,32 @@
 import { mergeWorkbenchExecutionTracesByJob } from "./execution-traces.js";
-export function buildSubjectCasePhaseRefs(args) {
+export function buildSubjectCaseExecutionRefs(args) {
     const groups = new Map();
     for (const job of args.jobs) {
-        const phase = readWorkbenchExecutionPurpose(job);
+        const kind = readWorkbenchExecutionPurpose(job);
         const jobSubjectId = job.subjectId ?? readWorkbenchExecutionMetadataString(job, "subjectId");
         const jobCaseId = readWorkbenchExecutionMetadataString(job, "caseId");
         if (jobSubjectId === args.subjectId &&
-            phase === "attempt" &&
+            kind === "attempt" &&
             caseReviewCaseIdsMatch(jobCaseId, args.caseId) &&
             caseReviewSampleIndicesMatch(readWorkbenchExecutionMetadataNumber(job, "sampleIndex"), args.sampleIndex)) {
             const key = [
                 job.runId,
-                phase,
+                kind,
                 jobCaseId ?? "",
                 readWorkbenchExecutionMetadataNumber(job, "sampleIndex") ?? "",
             ].join("\0");
             groups.set(key, [...(groups.get(key) ?? []), job]);
         }
     }
-    const phases = [...groups.values()]
-        .map((group) => group.slice().sort(compareWorkbenchPhaseJobs))
+    const executions = [...groups.values()]
+        .map((group) => group.slice().sort(compareWorkbenchExecutionJobs))
         .flatMap((group) => {
         const first = group[0];
         if (!first) {
             return [];
         }
-        const phase = readWorkbenchExecutionPurpose(first);
-        if (phase !== "attempt") {
+        const kind = readWorkbenchExecutionPurpose(first);
+        if (kind !== "attempt") {
             return [];
         }
         const startedAt = minTimestamp(group.map((job) => job.startedAt));
@@ -36,23 +36,32 @@ export function buildSubjectCasePhaseRefs(args) {
             : null;
         return [{
                 runId: first.runId,
-                phase,
+                kind,
                 role: "engine",
                 status: resolveWorkbenchJobGroupStatus(group),
                 jobIds: group.map((job) => job.id),
+                executionIds: group.flatMap((job) => {
+                    const executionId = readWorkbenchExecutionId(job);
+                    return executionId ? [executionId] : [];
+                }),
                 createdAt: minTimestamp(group.map((job) => job.createdAt)) ?? first.createdAt,
                 ...(startedAt ? { startedAt } : {}),
                 ...(finishedAt ? { finishedAt } : {}),
                 ...(durationMs !== null ? { durationMs } : {}),
+                ...optionalString("caseId", readWorkbenchExecutionMetadataString(first, "caseId")),
                 ...optionalNumber("sampleIndex", readWorkbenchExecutionMetadataNumber(first, "sampleIndex")),
+                ...optionalNumber("attemptIndex", readWorkbenchExecutionMetadataNumber(first, "attemptIndex")),
             }];
     })
-        .sort(compareSubjectCasePhases);
-    return selectCurrentPhaseRun(phases);
+        .sort(compareSubjectCaseExecutions);
+    return selectCurrentExecutionRun(executions);
 }
-export function buildWorkbenchTracePhases(args) {
+export function buildWorkbenchExecutionEvidence(args) {
     const groups = new Map();
     for (const job of args.jobs) {
+        if (isBaselineMaterializationJob(job)) {
+            continue;
+        }
         const purpose = readWorkbenchExecutionPurpose(job);
         if (!purpose) {
             continue;
@@ -79,27 +88,52 @@ export function buildWorkbenchTracePhases(args) {
             return [];
         }
         const role = traceRoleForPurpose(purpose);
+        const sessions = group.flatMap((job) => args.traceSessionsForJob
+            ? args.traceSessionsForJob(job, role)
+            : []);
+        const jobIds = group.map((job) => job.id);
+        const executionIds = group.flatMap((job) => {
+            const executionId = readWorkbenchExecutionId(job);
+            return executionId ? [executionId] : [];
+        });
         return [{
-                phase: purpose,
+                id: [
+                    purpose,
+                    first.runId,
+                    readWorkbenchExecutionMetadataString(first, "caseId") ?? "current",
+                    readWorkbenchExecutionMetadataNumber(first, "sampleIndex") ?? "sample",
+                    readWorkbenchExecutionMetadataNumber(first, "attemptIndex") ?? "attempt",
+                    jobIds.join("_"),
+                ].join(":"),
+                kind: purpose,
                 executionId: group.length === 1 ? readWorkbenchExecutionId(first) : null,
                 role,
                 status: resolveWorkbenchJobGroupStatus(group),
-                jobIds: group.map((job) => job.id),
+                jobIds,
+                executionIds,
                 ...(first.subjectId ? { subjectId: first.subjectId } : {}),
                 ...optionalString("caseId", readWorkbenchExecutionMetadataString(first, "caseId")),
                 ...optionalNumber("sampleIndex", readWorkbenchExecutionMetadataNumber(first, "sampleIndex")),
                 ...optionalNumber("attemptIndex", readWorkbenchExecutionMetadataNumber(first, "attemptIndex")),
+                sessions,
                 trace: mergeWorkbenchExecutionTracesByJob({
                     traceIdPrefix: args.traceIdPrefix,
                     stageId: purpose,
-                    jobs: group.map((job) => ({
-                        id: job.id,
-                        trace: args.traceForJob(job, role),
-                    })),
+                    jobs: [
+                        ...group.map((job) => ({
+                            id: job.id,
+                            trace: args.traceForJob(job, role),
+                        })),
+                        ...sessions.map((session) => ({
+                            id: session.id,
+                            jobId: session.jobId,
+                            trace: session.trace,
+                        })),
+                    ],
                 }),
             }];
     })
-        .sort(compareWorkbenchTracePhases);
+        .sort(compareWorkbenchExecutionEvidence);
 }
 export function readWorkbenchExecutionPurpose(job) {
     if (job.kind !== "execute") {
@@ -122,8 +156,8 @@ export function readWorkbenchExecutionMetadataNumber(job, key) {
     const raw = readWorkbenchExecutionMetadataValue(job, key);
     return typeof raw === "number" && Number.isFinite(raw) ? raw : null;
 }
-export function isWorkbenchPhaseActive(phase) {
-    return phase.status === "queued" || phase.status === "running";
+export function isWorkbenchExecutionActive(execution) {
+    return execution.status === "queued" || execution.status === "running";
 }
 export function resolveWorkbenchJobGroupStatus(jobs) {
     if (jobs.some((job) => job.status === "running")) {
@@ -150,44 +184,50 @@ function readExecutionRecord(job) {
     const input = asRecord(job.input);
     return asRecord(input?.execution);
 }
-function caseReviewCaseIdsMatch(jobCaseId, reviewCaseId) {
+function isBaselineMaterializationJob(job) {
+    const input = asRecord(job.input);
+    const execution = asRecord(input?.execution);
+    const metadata = asRecord(execution?.metadata);
+    return metadata?.baseline === true || input?.baseline === true;
+}
+function caseReviewCaseIdsMatch(jobCaseId, requestedCaseId) {
     return Boolean(jobCaseId) &&
-        (jobCaseId === reviewCaseId || reviewCaseId.startsWith(`${jobCaseId}__`));
+        (jobCaseId === requestedCaseId || requestedCaseId.startsWith(`${jobCaseId}__`));
 }
 function caseReviewSampleIndicesMatch(jobSampleIndex, reviewSampleIndex) {
     return typeof reviewSampleIndex !== "number" || jobSampleIndex === reviewSampleIndex;
 }
-function selectCurrentPhaseRun(phases) {
-    if (phases.length <= 1) {
-        return phases;
+function selectCurrentExecutionRun(executions) {
+    if (executions.length <= 1) {
+        return executions;
     }
-    const activeRunId = phases
-        .filter(isWorkbenchPhaseActive)
-        .sort(comparePhaseRecency)[0]?.runId;
-    const selectedRunId = activeRunId ?? phases.slice().sort(comparePhaseRecency)[0]?.runId;
+    const activeRunId = executions
+        .filter(isWorkbenchExecutionActive)
+        .sort(compareExecutionRecency)[0]?.runId;
+    const selectedRunId = activeRunId ?? executions.slice().sort(compareExecutionRecency)[0]?.runId;
     return selectedRunId
-        ? phases.filter((phase) => phase.runId === selectedRunId)
-        : phases;
+        ? executions.filter((execution) => execution.runId === selectedRunId)
+        : executions;
 }
-function compareSubjectCasePhases(left, right) {
-    return (phasePurposeOrder(left.phase) - phasePurposeOrder(right.phase) ||
+function compareSubjectCaseExecutions(left, right) {
+    return (executionKindOrder(left.kind) - executionKindOrder(right.kind) ||
         (left.sampleIndex ?? -1) - (right.sampleIndex ?? -1) ||
-        readPhaseRecencyMs(right) - readPhaseRecencyMs(left));
+        readExecutionRecencyMs(right) - readExecutionRecencyMs(left));
 }
-function comparePhaseRecency(left, right) {
-    return readPhaseRecencyMs(right) - readPhaseRecencyMs(left);
+function compareExecutionRecency(left, right) {
+    return readExecutionRecencyMs(right) - readExecutionRecencyMs(left);
 }
-function compareWorkbenchPhaseJobs(left, right) {
-    return (phasePurposeOrder(readWorkbenchExecutionPurpose(left)) -
-        phasePurposeOrder(readWorkbenchExecutionPurpose(right)) ||
+function compareWorkbenchExecutionJobs(left, right) {
+    return (executionKindOrder(readWorkbenchExecutionPurpose(left)) -
+        executionKindOrder(readWorkbenchExecutionPurpose(right)) ||
         (readWorkbenchExecutionMetadataNumber(left, "sampleIndex") ?? -1) -
             (readWorkbenchExecutionMetadataNumber(right, "sampleIndex") ?? -1) ||
         readJobRecencyMs(right) - readJobRecencyMs(left) ||
         left.id.localeCompare(right.id));
 }
 function compareWorkbenchTraceJobs(left, right) {
-    return (phasePurposeOrder(readWorkbenchExecutionPurpose(left)) -
-        phasePurposeOrder(readWorkbenchExecutionPurpose(right)) ||
+    return (executionKindOrder(readWorkbenchExecutionPurpose(left)) -
+        executionKindOrder(readWorkbenchExecutionPurpose(right)) ||
         String(readWorkbenchExecutionMetadataString(left, "caseId") ?? "").localeCompare(String(readWorkbenchExecutionMetadataString(right, "caseId") ?? "")) ||
         (readWorkbenchExecutionMetadataNumber(left, "sampleIndex") ?? -1) -
             (readWorkbenchExecutionMetadataNumber(right, "sampleIndex") ?? -1) ||
@@ -195,8 +235,8 @@ function compareWorkbenchTraceJobs(left, right) {
             (readWorkbenchExecutionMetadataNumber(right, "attemptIndex") ?? -1) ||
         left.id.localeCompare(right.id));
 }
-function compareWorkbenchTracePhases(left, right) {
-    return (phasePurposeOrder(left.phase) - phasePurposeOrder(right.phase) ||
+function compareWorkbenchExecutionEvidence(left, right) {
+    return (executionKindOrder(left.kind) - executionKindOrder(right.kind) ||
         String(left.caseId ?? "").localeCompare(String(right.caseId ?? "")) ||
         (left.sampleIndex ?? -1) - (right.sampleIndex ?? -1) ||
         (left.attemptIndex ?? -1) - (right.attemptIndex ?? -1) ||
@@ -208,19 +248,19 @@ function traceRoleForPurpose(purpose) {
     }
     return "engine";
 }
-function phasePurposeOrder(purpose) {
-    if (purpose === "improve") {
+function executionKindOrder(kind) {
+    if (kind === "improve") {
         return 0;
     }
-    if (purpose === "attempt") {
+    if (kind === "attempt") {
         return 1;
     }
     return 3;
 }
-function readPhaseRecencyMs(phase) {
-    return (parseTimestampMs(phase.finishedAt) ??
-        parseTimestampMs(phase.startedAt) ??
-        parseTimestampMs(phase.createdAt) ??
+function readExecutionRecencyMs(execution) {
+    return (parseTimestampMs(execution.finishedAt) ??
+        parseTimestampMs(execution.startedAt) ??
+        parseTimestampMs(execution.createdAt) ??
         0);
 }
 function readJobRecencyMs(job) {

package/dist/execution-graph.js

@@ -80,8 +80,7 @@ function createExecution(args) {
         subjectId: args.input.subjectId,
         purpose: args.purpose,
         adapter: args.adapter,
-        sandbox: args.input.environmentRef &&
-            runtimeSandboxRef(args.runtime) === runtimeSandboxRef(args.input.spec.environment)
+        sandbox: args.input.environmentRef
             ? {
                 kind: "oci",
                 ref: args.input.environmentRef,

package/dist/execution-jobs.js

@@ -76,7 +76,7 @@ export function engineCaseIds(engineCases) {
 export function engineCaseForCase(engineCases, caseId) {
     const engineCase = engineCases.find((bundle) => bundle.id === caseId);
     if (!engineCase) {
-        throw new Error(`Task bundle not found for case ${caseId}.`);
+        throw new Error(`Engine case not found for case ${caseId}.`);
     }
     return engineCase;
 }

package/dist/execution-outputs.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execution-outputs.d.ts","sourceRoot":"","sources":["../src/execution-outputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,IAAI,EAEJ,qBAAqB,EAErB,sBAAsB,EACtB,eAAe,EAChB,MAAM,kCAAkC,CAAC;AAI1C,MAAM,WAAW,gCAAgC;IAC/C,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC,MAAM,CAAC,EAAE,eAAe,CAAC;CAC1B;AAED,wBAAgB,wCAAwC,CACtD,SAAS,EAAE,sBAAsB,EACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAC7B,gCAAgC,CAoClC;AAED,wBAAgB,wCAAwC,CAAC,SAAS,EAAE,sBAAsB,GAAG,MAAM,EAAE,CAoFpG;AAsBD,wBAAgB,iCAAiC,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAKzF"}
+{"version":3,"file":"execution-outputs.d.ts","sourceRoot":"","sources":["../src/execution-outputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,IAAI,EAEJ,qBAAqB,EAErB,sBAAsB,EACtB,eAAe,EAChB,MAAM,kCAAkC,CAAC;AAI1C,MAAM,WAAW,gCAAgC;IAC/C,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC,MAAM,CAAC,EAAE,eAAe,CAAC;CAC1B;AAED,wBAAgB,wCAAwC,CACtD,SAAS,EAAE,sBAAsB,EACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAC7B,gCAAgC,CAoClC;AAED,wBAAgB,wCAAwC,CAAC,SAAS,EAAE,sBAAsB,GAAG,MAAM,EAAE,CAkFpG;AAsBD,wBAAgB,iCAAiC,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAKzF"}

package/dist/execution-outputs.js

@@ -87,6 +87,7 @@ export function collectWorkbenchExecutionIsolationIssues(execution) {
     }
     const outputNames = new Set();
     const expectedOutput = expectedOutputForPurpose(execution.purpose);
+    const isRuntimeControlExecution = execution.metadata.runtimeControl === true;
     for (const output of execution.outputs) {
         if (outputNames.has(output.name)) {
             issues.push(`Execution ${execution.id} declares duplicate output ${output.name}.`);
@@ -102,7 +103,7 @@ export function collectWorkbenchExecutionIsolationIssues(execution) {
             issues.push(`Execution ${execution.id} cannot declare ${output.schema} for purpose ${execution.purpose}.`);
         }
     }
-    if (expectedOutput !== null && !outputNames.has(expectedOutput)) {
+    if (expectedOutput !== null && !outputNames.has(expectedOutput) && !isRuntimeControlExecution) {
         issues.push(`Execution ${execution.id} missing required output ${expectedOutput} for purpose ${execution.purpose}.`);
     }
     const resources = execution.policy.resources;
@@ -111,9 +112,6 @@ export function collectWorkbenchExecutionIsolationIssues(execution) {
             issues.push(`Execution ${execution.id} policy.resources.${name} must be a positive number.`);
         }
     }
-    if (execution.policy.network.egress === "allowlist" && (!execution.policy.network.allow || execution.policy.network.allow.length === 0)) {
-        issues.push(`Execution ${execution.id} allowlist network policy must include at least one allowed host.`);
-    }
     return issues;
 }
 function expectedInputsForPurpose(purpose) {
@@ -271,13 +269,10 @@ function normalizeCaseCriteria(value, label, issues) {
         if (!record || !criterionId || score === null) {
             return [];
         }
-        const pass = record.pass === undefined
-            ? score >= 0.5
-            : typeof record.pass === "boolean"
-                ? record.pass
-                : null;
+        const pass = typeof record.pass === "boolean" ? record.pass : null;
         if (pass === null) {
             issues.push(`${itemLabel}.pass must be a boolean.`);
+            return [];
         }
         const errors = record.errors === undefined
             ? []
@@ -294,7 +289,7 @@ function normalizeCaseCriteria(value, label, issues) {
                 criterion_id: criterionId,
                 label: typeof record.label === "string" ? record.label : criterionId,
                 score,
-                pass: pass ?? false,
+                pass,
                 ...(errors && errors.length > 0 ? { errors } : {}),
                 ...(rationale ? { rationale } : {}),
             }];

package/dist/execution-runtime-types.d.ts

@@ -2,7 +2,7 @@ import type { HostedWorkbenchEnvironmentVersion, HostedWorkbenchJob, Json, Surfa
 import type { GenericRunSpec, WorkbenchEngineCase } from "./generic-spec.ts";
 import type { WorkbenchExecutionProgressTarget } from "./execution-events.ts";
 import type { WorkbenchAdapterAuthBundle } from "./adapter-auth.ts";
-import type { WorkbenchAdapterOperation, WorkbenchAdapterManifest } from "@workbench-ai/workbench-protocol";
+import type { WorkbenchAdapterOperation, WorkbenchAdapterOperationExecutor, WorkbenchAdapterManifest, WorkbenchRuntimeControlOperationSequenceRequest } from "@workbench-ai/workbench-protocol";
 export interface WorkbenchExecutionRuntimeInput {
     job: HostedWorkbenchJob;
     spec: GenericRunSpec;
@@ -11,6 +11,7 @@ export interface WorkbenchExecutionRuntimeInput {
     baseFiles: readonly SurfaceSnapshotFile[];
     engineResolveFiles: readonly SurfaceSnapshotFile[];
     engineCases: readonly WorkbenchEngineCase[];
+    adapterFiles?: readonly SurfaceSnapshotFile[];
     traceFiles?: readonly SurfaceSnapshotFile[];
     now?: string;
     adapterAuthProfiles?: readonly WorkbenchAdapterAuthBundle[];
@@ -18,16 +19,19 @@ export interface WorkbenchExecutionRuntimeInput {
     adapterAuthRoot?: string;
     adapterAuthRequest?: Json;
     adapterAuthEnv?: Record<string, string>;
+    adapterRuntimeEnv?: Record<string, string>;
     progress?: WorkbenchExecutionProgressTarget;
     runtimeRegistry?: string;
     pullImages?: boolean;
     workdir?: string;
     workspaceRoot?: string;
+    runtimeControlOperation?: WorkbenchRuntimeControlOperationSequenceRequest;
 }
-export interface WorkbenchWorkloadPhaseCommand {
-    kind: "optimizer" | "runner" | "engine";
+export interface WorkbenchWorkloadStepCommand {
+    kind: "optimizer" | "subject" | "engine";
     label: string;
     operation: WorkbenchAdapterOperation;
+    executor: WorkbenchAdapterOperationExecutor;
     adapter?: WorkbenchAdapterInvocation;
     command?: string;
     okExitCodes?: number[];

package/dist/execution-runtime-types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execution-runtime-types.d.ts","sourceRoot":"","sources":["../src/execution-runtime-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iCAAiC,EACjC,kBAAkB,EAClB,IAAI,EACJ,mBAAmB,EACnB,0BAA0B,EAC3B,MAAM,kCAAkC,CAAC;AAE1C,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,gCAAgC,EACjC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,0BAA0B,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,yBAAyB,EACzB,wBAAwB,EACzB,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,GAAG,EAAE,kBAAkB,CAAC;IACxB,IAAI,EAAE,cAAc,CAAC;IACrB,kBAAkB,CAAC,EAAE,IAAI,CAAC,iCAAiC,EAAE,IAAI,GAAG,UAAU,GAAG,YAAY,GAAG,MAAM,CAAC,CAAC;IACxG,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAS,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACnD,WAAW,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,UAAU,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,mBAAmB,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAC5D,gBAAgB,CAAC,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,QAAQ,CAAC,EAAE,gCAAgC,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,yBAAyB,CAAC;IACrC,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB"}
+{"version":3,"file":"execution-runtime-types.d.ts","sourceRoot":"","sources":["../src/execution-runtime-types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,iCAAiC,EACjC,kBAAkB,EAClB,IAAI,EACJ,mBAAmB,EACnB,0BAA0B,EAC3B,MAAM,kCAAkC,CAAC;AAE1C,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,gCAAgC,EACjC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,0BAA0B,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EACV,yBAAyB,EACzB,iCAAiC,EACjC,wBAAwB,EACxB,+CAA+C,EAChD,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,8BAA8B;IAC7C,GAAG,EAAE,kBAAkB,CAAC;IACxB,IAAI,EAAE,cAAc,CAAC;IACrB,kBAAkB,CAAC,EAAE,IAAI,CAAC,iCAAiC,EAAE,IAAI,GAAG,UAAU,GAAG,YAAY,GAAG,MAAM,CAAC,CAAC;IACxG,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAS,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACnD,WAAW,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,YAAY,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC9C,UAAU,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,mBAAmB,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAC5D,gBAAgB,CAAC,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kBAAkB,CAAC,EAAE,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,QAAQ,CAAC,EAAE,gCAAgC,CAAC;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,+CAA+C,CAAC;CAC3E;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,yBAAyB,CAAC;IACrC,QAAQ,EAAE,iCAAiC,CAAC;IAC5C,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB"}

package/dist/execution-traces.d.ts

@@ -1,6 +1,7 @@
-import type { HostedWorkbenchJob, WorkbenchExecutionTrace } from "@workbench-ai/workbench-contract";
+import type { HostedWorkbenchJob, SurfaceSnapshotFile, WorkbenchExecutionTrace, WorkbenchTraceSession } from "@workbench-ai/workbench-contract";
 export interface WorkbenchTraceMergeJob {
     id: string;
+    jobId?: string;
     trace: WorkbenchExecutionTrace;
 }
 export declare function finalizeWorkbenchExecutionTraceForJob(args: {
@@ -13,4 +14,13 @@ export declare function mergeWorkbenchExecutionTracesByJob(args: {
     stageId?: string | null;
     jobs: readonly WorkbenchTraceMergeJob[];
 }): WorkbenchExecutionTrace;
+export declare function buildWorkbenchTraceSessionsFromFiles(args: {
+    job: HostedWorkbenchJob;
+    files: readonly SurfaceSnapshotFile[];
+    purpose?: string | null;
+    fallbackRole: WorkbenchTraceSession["role"];
+}): WorkbenchTraceSession[];
+export declare function combineWorkbenchTraceSessions(sessions: readonly WorkbenchTraceSession[]): WorkbenchExecutionTrace;
+export declare function readWorkbenchExecutionTraceFiles(files: readonly SurfaceSnapshotFile[]): WorkbenchExecutionTrace | null;
+export declare function traceSessionLabel(filePath: string, role: WorkbenchTraceSession["role"]): string;
 //# sourceMappingURL=execution-traces.d.ts.map

package/dist/execution-traces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"execution-traces.d.ts","sourceRoot":"","sources":["../src/execution-traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAElB,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,uBAAuB,CAAC;CAChC;AAED,wBAAgB,qCAAqC,CAAC,IAAI,EAAE;IAC1D,GAAG,EAAE,kBAAkB,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,uBAAuB,CAAC;CAChC,GAAG,uBAAuB,CAsD1B;AAED,wBAAgB,kCAAkC,CAAC,IAAI,EAAE;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,SAAS,sBAAsB,EAAE,CAAC;CACzC,GAAG,uBAAuB,CAoC1B"}
+{"version":3,"file":"execution-traces.d.ts","sourceRoot":"","sources":["../src/execution-traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAElB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACtB,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,uBAAuB,CAAC;CAChC;AAED,wBAAgB,qCAAqC,CAAC,IAAI,EAAE;IAC1D,GAAG,EAAE,kBAAkB,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,uBAAuB,CAAC;CAChC,GAAG,uBAAuB,CAsD1B;AAED,wBAAgB,kCAAkC,CAAC,IAAI,EAAE;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,SAAS,sBAAsB,EAAE,CAAC;CACzC,GAAG,uBAAuB,CAqC1B;AAED,wBAAgB,oCAAoC,CAAC,IAAI,EAAE;IACzD,GAAG,EAAE,kBAAkB,CAAC;IACxB,KAAK,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,qBAAqB,CAAC,MAAM,CAAC,CAAC;CAC7C,GAAG,qBAAqB,EAAE,CA2B1B;AAED,wBAAgB,6BAA6B,CAC3C,QAAQ,EAAE,SAAS,qBAAqB,EAAE,GACzC,uBAAuB,CAOzB;AAED,wBAAgB,gCAAgC,CAC9C,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,uBAAuB,GAAG,IAAI,CAiBhC;AAED,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC,GAClC,MAAM,CAaR"}