@workbench-ai/workbench-core 0.0.46

package/dist/execution-graph.js

@@ -0,0 +1,126 @@
+import { resolveEngineCaseExecutionConfig, runtimeNetwork, runtimeResources, runtimeSandboxRef, } from "./generic-spec.js";
+export function compileWorkbenchExecutionGraph(input) {
+    const workflow = input.workflow ?? "improve";
+    const sampleIndex = input.sampleIndex ?? 0;
+    const caseId = input.caseId ?? "current";
+    const subjectRef = input.subjectRef ?? `workbench://benchmarks/${input.projectId}/subjects/${input.subjectId}`;
+    const caseRef = input.caseRef ?? `workbench://benchmarks/${input.projectId}/engine-cases/${caseId}`;
+    if (!input.engineCase) {
+        throw new Error("Execution graph compilation requires an engine case.");
+    }
+    const engineCase = input.engineCase;
+    const executionConfig = resolveEngineCaseExecutionConfig({
+        spec: input.spec,
+        engineCase,
+    });
+    const nodes = [];
+    const executions = [];
+    const optimizerExecutionId = executionId(input, "improve", "current", 0);
+    const optimizerOutputRef = `execution://${optimizerExecutionId}/subject_patch`;
+    const engineAdapter = input.spec.engineRun;
+    if (workflow === "improve") {
+        if (!input.spec.optimizer || !input.spec.improve) {
+            throw new Error("Optimizer YAML is required for improve execution graphs.");
+        }
+        pushExecution(nodes, executions, createExecution({
+            input,
+            purpose: "improve",
+            adapter: input.spec.improve,
+            inputs: [
+                inputRef("subject", subjectRef, "/workspace/input/subject", false),
+                inputRef("traces", `workbench://benchmarks/${input.projectId}/runs/${input.runId}/traces`, "/workspace/input/traces", false),
+            ],
+            outputs: [outputContract("subject_patch", "workbench.subject_patch.v1")],
+            metadata: {
+                attemptIndex: input.attemptIndex,
+                sampleIndex: 0,
+                caseId: "current",
+                benchmark: input.spec.benchmark.name,
+                edits: input.spec.optimizer.edits,
+            },
+            runtime: input.spec.environment,
+            idOverride: optimizerExecutionId,
+        }), []);
+    }
+    const runSubjectRef = workflow === "improve" ? optimizerOutputRef : subjectRef;
+    const attemptExecutionId = executionId(input, "attempt", caseId, sampleIndex);
+    pushExecution(nodes, executions, createExecution({
+        input,
+        purpose: "attempt",
+        adapter: engineAdapter,
+        inputs: [
+            inputRef("subject", runSubjectRef, "/workspace/input/subject", false),
+            inputRef("case", caseRef, "/workspace/input/case", false),
+        ],
+        outputs: [outputContract("result", "workbench.result.v1")],
+        metadata: {
+            attemptIndex: input.attemptIndex,
+            sampleIndex,
+            caseId,
+            engineCase: engineCase,
+            ...(executionConfig.environment.workdir ? { workdir: executionConfig.environment.workdir } : {}),
+        },
+        runtime: executionConfig.environment,
+        idOverride: attemptExecutionId,
+    }), workflow === "improve" ? [optimizerExecutionId] : []);
+    return { nodes, executions };
+}
+function pushExecution(nodes, executions, execution, dependsOn) {
+    nodes.push({
+        execution,
+        dependsOn,
+    });
+    executions.push(execution);
+}
+function createExecution(args) {
+    return {
+        id: args.idOverride ?? executionId(args.input, args.purpose, args.input.caseId ?? "current", args.input.sampleIndex ?? 0),
+        projectId: args.input.projectId,
+        runId: args.input.runId,
+        subjectId: args.input.subjectId,
+        purpose: args.purpose,
+        adapter: args.adapter,
+        sandbox: args.input.environmentRef &&
+            runtimeSandboxRef(args.runtime) === runtimeSandboxRef(args.input.spec.environment)
+            ? {
+                kind: "oci",
+                ref: args.input.environmentRef,
+            }
+            : {
+                kind: "oci",
+                ref: runtimeSandboxRef(args.runtime),
+            },
+        inputs: args.inputs,
+        outputs: args.outputs,
+        policy: executionPolicy(args.input.ownerUserId, args.runtime),
+        metadata: args.metadata,
+    };
+}
+function executionPolicy(tenantId, runtime) {
+    return {
+        tenantId,
+        resources: runtimeResources(runtime),
+        network: runtimeNetwork(runtime),
+    };
+}
+function executionId(input, purpose, caseId, sampleIndex, suffix) {
+    const caseKey = caseId.replace(/[^a-z0-9_]/giu, "_");
+    const parts = [
+        "exec",
+        input.runId.replace(/[^a-z0-9_]/giu, "_"),
+        `attempt_${String(input.attemptIndex).padStart(3, "0")}`,
+        `case_${caseKey}`,
+        `sample_${String(sampleIndex).padStart(3, "0")}`,
+        purpose,
+    ];
+    if (suffix) {
+        parts.push(suffix.replace(/[^a-z0-9_]/giu, "_"));
+    }
+    return parts.join("_");
+}
+function inputRef(name, ref, mountPath, writable) {
+    return { name, ref, mountPath, writable };
+}
+function outputContract(name, schema, required = true) {
+    return { name, schema, required };
+}

package/dist/execution-jobs.d.ts

@@ -0,0 +1,70 @@
+import type { HostedWorkbenchJob, Json, SurfaceSnapshotFile, WorkbenchExecutionSpec } from "@workbench-ai/workbench-contract";
+import type { GenericRunSpec, WorkbenchEngineCase } from "./generic-spec.ts";
+export type WorkbenchRunWorkflow = "eval" | "improve";
+export declare const MAX_WORKBENCH_RUN_BUDGET = 20;
+export declare function expectedWorkbenchRunJobCount(args: {
+    workflow: WorkbenchRunWorkflow;
+    budget: number;
+    samples: number;
+    caseCount: number;
+}): number;
+export declare function validateWorkbenchRunEnvelope(args: {
+    workflow: WorkbenchRunWorkflow;
+    budget: number;
+    samples: number;
+    caseCount: number;
+}): string | null;
+export declare function attemptJobCountForRunSpec(_spec: GenericRunSpec): number;
+export declare function planWorkbenchExecutionJobsForPurpose(args: {
+    ownerUserId: string;
+    projectId: string;
+    runId: string;
+    subjectId: string;
+    attemptIndex: number;
+    samples: number;
+    caseIds?: readonly string[];
+    spec: GenericRunSpec;
+    workflow: WorkbenchRunWorkflow;
+    purpose: WorkbenchExecutionSpec["purpose"];
+    now: string;
+    baseFiles?: readonly SurfaceSnapshotFile[];
+    engineCases: readonly WorkbenchEngineCase[];
+    traceFiles?: readonly SurfaceSnapshotFile[];
+    environmentRef?: string;
+    environmentRefsByCase?: ReadonlyMap<string, string>;
+    baseId?: string | null;
+}): HostedWorkbenchJob[];
+export declare function engineCaseIds(engineCases: readonly WorkbenchEngineCase[]): string[];
+export declare function engineCaseForCase(engineCases: readonly WorkbenchEngineCase[], caseId: string): WorkbenchEngineCase;
+export declare function createWorkbenchExecutionJob(args: {
+    projectId: string;
+    runId: string;
+    subjectId: string;
+    execution: WorkbenchExecutionSpec;
+    dependsOn: readonly string[];
+    now: string;
+    baseFiles?: readonly SurfaceSnapshotFile[];
+    traceFiles?: readonly SurfaceSnapshotFile[];
+    baseId?: string | null;
+}): HostedWorkbenchJob;
+export declare function createBaselineSubjectExecution(args: {
+    ownerUserId: string;
+    projectId: string;
+    runId: string;
+    subjectId: string;
+    attemptIndex: number;
+}): WorkbenchExecutionSpec;
+export declare function createBaselineSubjectJob(args: {
+    ownerUserId: string;
+    projectId: string;
+    runId: string;
+    subjectId: string;
+    files: readonly SurfaceSnapshotFile[];
+    now: string;
+    baseId: string | null;
+    attemptIndex: number;
+    fileSet?: Json;
+}): HostedWorkbenchJob;
+export declare function workbenchExecutionJobId(executionId: string): string;
+export declare function workbenchExecutionJobPurpose(job: HostedWorkbenchJob): WorkbenchExecutionSpec["purpose"] | null;
+//# sourceMappingURL=execution-jobs.d.ts.map

package/dist/execution-jobs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-jobs.d.ts","sourceRoot":"","sources":["../src/execution-jobs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,IAAI,EACJ,mBAAmB,EACnB,sBAAsB,EACvB,MAAM,kCAAkC,CAAC;AAK1C,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,SAAS,CAAC;AAEtD,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,wBAAgB,4BAA4B,CAAC,IAAI,EAAE;IACjD,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,MAAM,CAMT;AAED,wBAAgB,4BAA4B,CAAC,IAAI,EAAE;IACjD,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,MAAM,GAAG,IAAI,CAchB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM,CAEvE;AAED,wBAAgB,oCAAoC,CAAC,IAAI,EAAE;IACzD,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,cAAc,CAAC;IACrB,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,sBAAsB,CAAC,SAAS,CAAC,CAAC;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC3C,WAAW,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,UAAU,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qBAAqB,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GAAG,kBAAkB,EAAE,CA4CvB;AAED,wBAAgB,aAAa,CAAC,WAAW,EAAE,SAAS,mBAAmB,EAAE,GAAG,MAAM,EAAE,CAEnF;AAED,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,SAAS,mBAAmB,EAAE,EAC3C,MAAM,EAAE,MAAM,GACb,mBAAmB,CAMrB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,sBAAsB,CAAC;IAClC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC3C,UAAU,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IAC5C,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GAAG,kBAAkB,CA0BrB;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,sBAAsB,CAwCzB;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE;IAC7C,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,IAAI,CAAC;CAChB,GAAG,kBAAkB,CA6CrB;AAED,wBAAgB,uBAAuB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAEnE;AAED,wBAAgB,4BAA4B,CAAC,GAAG,EAAE,kBAAkB,GAAG,sBAAsB,CAAC,SAAS,CAAC,GAAG,IAAI,CAO9G"}

package/dist/execution-jobs.js

@@ -0,0 +1,229 @@
+import { compileWorkbenchExecutionGraph, } from "./execution-graph.js";
+export const MAX_WORKBENCH_RUN_BUDGET = 20;
+export function expectedWorkbenchRunJobCount(args) {
+    const caseCount = Math.max(1, Math.floor(args.caseCount));
+    if (args.workflow === "improve") {
+        return args.budget * (1 + (args.samples * caseCount));
+    }
+    return 1 + (args.samples * caseCount);
+}
+export function validateWorkbenchRunEnvelope(args) {
+    if (!Number.isSafeInteger(args.budget) || args.budget <= 0) {
+        return "Run budget must be a positive integer.";
+    }
+    if (!Number.isSafeInteger(args.samples) || args.samples <= 0) {
+        return "Run samples must be a positive integer.";
+    }
+    if (!Number.isSafeInteger(args.caseCount) || args.caseCount <= 0) {
+        return "Run case count must be a positive integer.";
+    }
+    if (args.budget > MAX_WORKBENCH_RUN_BUDGET) {
+        return `Run budget cannot exceed ${MAX_WORKBENCH_RUN_BUDGET}.`;
+    }
+    return null;
+}
+export function attemptJobCountForRunSpec(_spec) {
+    return 1;
+}
+export function planWorkbenchExecutionJobsForPurpose(args) {
+    const jobs = [];
+    const engineCases = args.engineCases;
+    const caseIds = args.caseIds && args.caseIds.length > 0
+        ? [...args.caseIds]
+        : engineCaseIds(engineCases);
+    if (caseIds.length === 0) {
+        throw new Error("Run planning requires at least one engine case.");
+    }
+    for (const caseId of caseIds) {
+        const engineCase = engineCaseForCase(engineCases, caseId);
+        for (let sampleIndex = 0; sampleIndex < args.samples; sampleIndex += 1) {
+            const graph = compileWorkbenchExecutionGraph({
+                ownerUserId: args.ownerUserId,
+                projectId: args.projectId,
+                runId: args.runId,
+                subjectId: args.subjectId,
+                attemptIndex: args.attemptIndex,
+                sampleIndex,
+                caseId,
+                spec: args.spec,
+                engineCase: engineCase.case,
+                environmentRef: args.environmentRefsByCase?.get(caseId) ?? args.environmentRef,
+                workflow: args.workflow === "improve" ? "improve" : "eval",
+            });
+            for (const node of graph.nodes) {
+                if (node.execution.purpose !== args.purpose) {
+                    continue;
+                }
+                jobs.push(createWorkbenchExecutionJob({
+                    projectId: args.projectId,
+                    runId: args.runId,
+                    subjectId: args.subjectId,
+                    execution: node.execution,
+                    dependsOn: node.dependsOn,
+                    now: args.now,
+                    ...(args.baseFiles ? { baseFiles: args.baseFiles } : {}),
+                    ...(args.traceFiles ? { traceFiles: args.traceFiles } : {}),
+                    ...(args.baseId ? { baseId: args.baseId } : {}),
+                }));
+            }
+        }
+    }
+    return jobs.filter((job, index) => jobs.findIndex((entry) => entry.id === job.id) === index);
+}
+export function engineCaseIds(engineCases) {
+    return [...new Set(engineCases.map((bundle) => bundle.id))].sort();
+}
+export function engineCaseForCase(engineCases, caseId) {
+    const engineCase = engineCases.find((bundle) => bundle.id === caseId);
+    if (!engineCase) {
+        throw new Error(`Task bundle not found for case ${caseId}.`);
+    }
+    return engineCase;
+}
+export function createWorkbenchExecutionJob(args) {
+    const attemptIndex = readExecutionMetadataNumber(args.execution, "attemptIndex");
+    const sampleIndex = readExecutionMetadataNumber(args.execution, "sampleIndex");
+    const caseId = readExecutionMetadataString(args.execution, "caseId");
+    return {
+        id: workbenchExecutionJobId(args.execution.id),
+        projectId: args.projectId,
+        runId: args.runId,
+        subjectId: args.subjectId,
+        kind: "execute",
+        status: "queued",
+        attempt: 0,
+        createdAt: args.now,
+        updatedAt: args.now,
+        input: {
+            execution: args.execution,
+            dependsOn: args.dependsOn.map(workbenchExecutionJobId),
+            subjectId: args.subjectId,
+            attemptIndex,
+            sampleIndex,
+            caseId,
+            ...(args.baseFiles ? { baseFiles: args.baseFiles.map((file) => ({ ...file })) } : {}),
+            ...(args.traceFiles ? { traceFiles: args.traceFiles.map((file) => ({ ...file })) } : {}),
+            ...(args.baseId ? { baseId: args.baseId } : {}),
+        },
+    };
+}
+export function createBaselineSubjectExecution(args) {
+    return {
+        id: `exec_${args.runId.replace(/[^a-z0-9_]/giu, "_")}_attempt_${String(args.attemptIndex).padStart(3, "0")}_case_current_sample_000_improve`,
+        projectId: args.projectId,
+        runId: args.runId,
+        subjectId: args.subjectId,
+        purpose: "improve",
+        adapter: {
+            use: "baseline",
+            with: {},
+        },
+        sandbox: {
+            kind: "snapshot",
+            ref: "workbench/baseline-subject",
+        },
+        inputs: [],
+        outputs: [{
+                name: "subject_patch",
+                schema: "workbench.subject_patch.v1",
+                required: true,
+            }],
+        policy: {
+            tenantId: args.ownerUserId,
+            resources: {
+                cpu: 1,
+                memoryGb: 1,
+                diskGb: 1,
+                timeoutMinutes: 1,
+            },
+            network: {
+                egress: "none",
+            },
+        },
+        metadata: {
+            attemptIndex: args.attemptIndex,
+            sampleIndex: 0,
+            caseId: "current",
+            baseline: true,
+        },
+    };
+}
+export function createBaselineSubjectJob(args) {
+    const execution = createBaselineSubjectExecution({
+        ownerUserId: args.ownerUserId,
+        projectId: args.projectId,
+        runId: args.runId,
+        subjectId: args.subjectId,
+        attemptIndex: args.attemptIndex,
+    });
+    const files = args.files.map((file) => ({ ...file }));
+    return {
+        id: workbenchExecutionJobId(execution.id),
+        projectId: args.projectId,
+        runId: args.runId,
+        subjectId: args.subjectId,
+        kind: "execute",
+        status: "succeeded",
+        attempt: 1,
+        createdAt: args.now,
+        startedAt: args.now,
+        finishedAt: args.now,
+        updatedAt: args.now,
+        input: {
+            execution,
+            dependsOn: [],
+            subjectId: args.subjectId,
+            attemptIndex: args.attemptIndex,
+            baseline: true,
+        },
+        output: {
+            ok: true,
+            executionId: execution.id,
+            purpose: "improve",
+            subjectId: args.subjectId,
+            attemptIndex: args.attemptIndex,
+            baseId: args.baseId,
+            subjectPatch: {
+                files,
+                fileChanges: [],
+            },
+            fileChanges: [],
+            files,
+            ...(args.fileSet ? { fileSet: args.fileSet } : {}),
+            traces: [],
+        },
+    };
+}
+export function workbenchExecutionJobId(executionId) {
+    return `job_${executionId.replace(/[^a-z0-9_]/giu, "_")}`;
+}
+export function workbenchExecutionJobPurpose(job) {
+    if (job.kind !== "execute") {
+        return null;
+    }
+    const execution = asRecord(asRecord(job.input).execution);
+    const purpose = execution.purpose;
+    return purpose === "improve" || purpose === "attempt" ? purpose : null;
+}
+function readExecutionMetadataNumber(execution, key) {
+    const raw = execution.metadata[key];
+    if (typeof raw === "number" && Number.isFinite(raw)) {
+        return raw;
+    }
+    throw new Error(`Execution ${execution.id} is missing numeric metadata.${key}.`);
+}
+function readExecutionMetadataString(execution, key) {
+    const raw = execution.metadata[key];
+    if (typeof raw === "string" && raw.length > 0) {
+        return raw;
+    }
+    throw new Error(`Execution ${execution.id} is missing string metadata.${key}.`);
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+}
+function normalizeRelativePath(value) {
+    return value.trim().replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/\/+$/u, "");
+}

package/dist/execution-outputs.d.ts

@@ -0,0 +1,9 @@
+import type { Json, WorkbenchSubjectPatch, WorkbenchExecutionSpec, WorkbenchResult } from "@workbench-ai/workbench-contract";
+export interface WorkbenchExecutionOutputPayloads {
+    subjectPatch?: WorkbenchSubjectPatch;
+    result?: WorkbenchResult;
+}
+export declare function validateWorkbenchExecutionOutputPayloads(execution: WorkbenchExecutionSpec, payloads: Record<string, Json>): WorkbenchExecutionOutputPayloads;
+export declare function collectWorkbenchExecutionIsolationIssues(execution: WorkbenchExecutionSpec): string[];
+export declare function assertWorkbenchExecutionIsolation(execution: WorkbenchExecutionSpec): void;
+//# sourceMappingURL=execution-outputs.d.ts.map

package/dist/execution-outputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-outputs.d.ts","sourceRoot":"","sources":["../src/execution-outputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,IAAI,EAEJ,qBAAqB,EAErB,sBAAsB,EACtB,eAAe,EAChB,MAAM,kCAAkC,CAAC;AAI1C,MAAM,WAAW,gCAAgC;IAC/C,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC,MAAM,CAAC,EAAE,eAAe,CAAC;CAC1B;AAED,wBAAgB,wCAAwC,CACtD,SAAS,EAAE,sBAAsB,EACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAC7B,gCAAgC,CAoClC;AAED,wBAAgB,wCAAwC,CAAC,SAAS,EAAE,sBAAsB,GAAG,MAAM,EAAE,CAoFpG;AAsBD,wBAAgB,iCAAiC,CAAC,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAKzF"}