@workbench-ai/workbench-core 0.0.49 → 0.0.51

package/dist/execution-usage.js

@@ -12,7 +12,7 @@ const NUMERIC_USAGE_FIELDS = [
     "costUsd",
 ];
 const USAGE_ROLES = [
-    "optimizer",
+    "improver",
     "runner",
     "engine",
 ];
@@ -44,18 +44,18 @@ export function completeUsageSummary(usage) {
     if (!usage) {
         return undefined;
     }
-    const optimizer = usage.optimizer ? normalizeExecutionUsage(usage.optimizer) : undefined;
+    const improver = usage.improver ? normalizeExecutionUsage(usage.improver) : undefined;
     const runner = usage.runner ? normalizeExecutionUsage(usage.runner) : undefined;
     const engine = usage.engine ? normalizeExecutionUsage(usage.engine) : undefined;
     const roleTotal = mergeExecutionUsage([
-        optimizer,
+        improver,
         runner,
         engine,
     ]);
     const total = roleTotal ?? normalizeExecutionUsage(usage.total);
     return compactUsageSummary({
         ...(total ? { total } : {}),
-        ...(optimizer ? { optimizer } : {}),
+        ...(improver ? { improver } : {}),
         ...(runner ? { runner } : {}),
         ...(engine ? { engine } : {}),
     });
@@ -63,12 +63,12 @@ export function completeUsageSummary(usage) {
 export function normalizeUsageSummary(value) {
     const record = jsonRecord(value);
     const total = normalizeExecutionUsage(record.total);
-    const optimizer = normalizeExecutionUsage(record.optimizer);
+    const improver = normalizeExecutionUsage(record.improver);
     const runner = normalizeExecutionUsage(record.runner);
     const engine = normalizeExecutionUsage(record.engine);
     return completeUsageSummary({
         ...(total ? { total } : {}),
-        ...(optimizer ? { optimizer } : {}),
+        ...(improver ? { improver } : {}),
         ...(runner ? { runner } : {}),
         ...(engine ? { engine } : {}),
     });
@@ -83,17 +83,17 @@ export function mergeUsageSummaries(summaries) {
     }
     return compactUsageSummary({
         total: mergeExecutionUsage(entries.map((entry) => entry.total)),
-        optimizer: mergeExecutionUsage(entries.map((entry) => entry.optimizer)),
+        improver: mergeExecutionUsage(entries.map((entry) => entry.improver)),
         runner: mergeExecutionUsage(entries.map((entry) => entry.runner)),
         engine: mergeExecutionUsage(entries.map((entry) => entry.engine)),
     });
 }
 export function mergeUsageRoles(roles) {
-    const optimizer = completeUsageSummary(roles.optimizer);
+    const improver = completeUsageSummary(roles.improver);
     const runner = completeUsageSummary(roles.runner);
     const engine = completeUsageSummary(roles.engine);
     return completeUsageSummary({
-        optimizer: optimizer?.optimizer ?? optimizer?.total,
+        improver: improver?.improver ?? improver?.total,
         runner: runner?.runner ?? runner?.total,
         engine: engine?.engine ?? engine?.total,
     });

package/dist/generic-spec.d.ts

@@ -1,6 +1,7 @@
 import { type EngineResolveBinding, type SurfaceSnapshotFile, type WorkbenchAdapterInvocation, type WorkbenchExecutionNetworkPolicy, type WorkbenchExecutionResources, type WorkbenchSpecValidation } from "@workbench-ai/workbench-contract";
 import type { WorkbenchEngineCase, WorkbenchEngineCaseSpec } from "@workbench-ai/workbench-protocol";
 export declare const BENCHMARK_SPEC_FILE = "benchmark.yaml";
+export declare const CANDIDATE_SPEC_FILE = "candidate.yaml";
 export interface WorkbenchRuntimeSpec {
     dockerfile: string;
     workdir?: string;
@@ -15,42 +16,53 @@ export interface WorkbenchRuntimeSpec {
 export interface WorkbenchPathRef {
     path: string;
 }
-export interface WorkbenchSubjectPrepareSpec {
+export interface WorkbenchCandidatePrepareSpec {
     command: string;
 }
 export interface AuthoredBenchmarkSpec {
-    version: 3;
+    version: 4;
     name: string;
     description: string;
     adapters: string[];
     engine: WorkbenchAdapterInvocation;
 }
-export interface WorkbenchSubjectManifestSpec {
-    version: 3;
+export interface WorkbenchCandidateRunSpec extends WorkbenchAdapterInvocation {
     name: string;
-    description?: string;
-    files: WorkbenchPathRef;
-    prepare?: WorkbenchSubjectPrepareSpec;
-    adapters: string[];
-    run: WorkbenchAdapterInvocation;
 }
-export type ResolvedSubjectSpec = WorkbenchSubjectManifestSpec;
-export interface AuthoredOptimizerSpec {
-    version: 3;
+export interface WorkbenchCaseSelector {
+    all?: true;
+    split?: string;
+}
+export interface WorkbenchSelectionSpec {
+    metric: string;
+    cases?: WorkbenchCaseSelector;
+}
+export interface WorkbenchCandidateImproveSpec extends WorkbenchAdapterInvocation {
+    edits: string[];
+    optimizeOn?: WorkbenchCaseSelector;
+    selectBy?: WorkbenchSelectionSpec;
+}
+export interface WorkbenchCandidateManifestSpec {
+    version: 4;
     name: string;
     description?: string;
-    edits: string[];
+    files: WorkbenchPathRef;
+    prepare?: WorkbenchCandidatePrepareSpec;
     adapters: string[];
-    improve: WorkbenchAdapterInvocation;
+    defaultRun?: string;
+    runs: Record<string, WorkbenchCandidateRunSpec>;
+    improve?: WorkbenchCandidateImproveSpec;
+}
+export interface ResolvedCandidateSpec extends WorkbenchCandidateManifestSpec {
+    selectedRunId: string;
 }
 export interface WorkbenchResolvedSource {
-    version: 3;
+    version: 4;
     benchmark: AuthoredBenchmarkSpec;
-    subject: ResolvedSubjectSpec;
-    optimizer?: AuthoredOptimizerSpec;
+    candidate: ResolvedCandidateSpec;
 }
 export interface GenericRunSpec {
-    version: 3;
+    version: 4;
     name: string;
     description: string;
     benchmark: {
@@ -58,16 +70,20 @@ export interface GenericRunSpec {
         description: string;
         engine: WorkbenchAdapterInvocation;
     };
-    subject: {
+    candidate: {
         name: string;
         description?: string;
         files: WorkbenchPathRef;
-        prepare?: WorkbenchSubjectPrepareSpec;
-    };
-    optimizer?: {
-        name: string;
-        description?: string;
-        edits: string[];
+        prepare?: WorkbenchCandidatePrepareSpec;
+        defaultRun: string;
+        selectedRunId: string;
+        selectedRunName: string;
+        runs: Record<string, WorkbenchCandidateRunSpec>;
+        improve?: {
+            edits: string[];
+            optimizeOn?: WorkbenchCaseSelector;
+            selectBy?: WorkbenchSelectionSpec;
+        };
     };
     environment: WorkbenchRuntimeSpec;
     adapters: string[];
@@ -91,16 +107,16 @@ export declare function engineResolveBindingForSourceYaml(source: string): Engin
 export declare function engineResolveBindingForSpec(spec: GenericRunSpec): EngineResolveBinding;
 export declare function resolveWorkbenchSourceFiles(args: {
     benchmarkSource: string;
-    subjectSource: string;
-    optimizerSource?: string | null;
+    candidateSource: string;
+    runId?: string | null;
 }): GenericRunSpec;
 export declare function parseWorkbenchSourceFiles(args: {
     benchmarkSource: string;
-    subjectSource?: string;
-    optimizerSource?: string | null;
+    candidateSource?: string;
+    runId?: string | null;
 }): WorkbenchResolvedSource;
 export declare function serializeWorkbenchResolvedSourceYaml(source: WorkbenchResolvedSource): string;
-export declare function isWorkbenchSubjectManifestPath(filePath: string): boolean;
+export declare function isWorkbenchCandidateManifestPath(filePath: string): boolean;
 export declare function resolveEngineCaseExecutionConfig(args: {
     spec: GenericRunSpec;
     engineCase: GenericEngineCaseSpec;

package/dist/generic-spec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generic-spec.d.ts","sourceRoot":"","sources":["../src/generic-spec.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,+BAA+B,EACpC,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC7B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAG1C,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAEpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE;QACV,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,+BAA+B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,gBAAgB,CAAC;IACxB,OAAO,CAAC,EAAE,2BAA2B,CAAC;IACtC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAED,MAAM,MAAM,mBAAmB,GAAG,4BAA4B,CAAC;AAE/D,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,0BAA0B,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,CAAC,CAAC;IACX,SAAS,EAAE,qBAAqB,CAAC;IACjC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,0BAA0B,CAAC;KACpC,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,gBAAgB,CAAC;QACxB,OAAO,CAAC,EAAE,2BAA2B,CAAC;KACvC,CAAC;IACF,SAAS,CAAC,EAAE;QACV,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,MAAM,EAAE,CAAC;KACjB,CAAC;IACF,WAAW,EAAE,oBAAoB,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;IACnC,aAAa,EAAE,0BAA0B,CAAC;IAC1C,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,GAAG,EAAE,0BAA0B,CAAC;IAChC,SAAS,EAAE,0BAA0B,CAAC;CACvC;AAED,MAAM,MAAM,qBAAqB,GAAG,uBAAuB,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,oBAAoB,CAAC;IAClC,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAQD,eAAO,MAAM,2BAA2B,EAAE,2BAKzC,CAAC;AAEF,wBAAgB,mCAAmC,CACjD,MAAM,EAAE,MAAM,GACb,uBAAuB,CAmBzB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,MAAM,GACb,cAAc,CAsChB;AAED,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,MAAM,GACb,oBAAoB,CAEtB;AAED,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,cAAc,GACnB,oBAAoB,CAStB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,cAAc,CAMjB;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC,GAAG,uBAAuB,CA4B1B;AAED,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,uBAAuB,GAC9B,MAAM,CAER;AAED,wBAAgB,8BAA8B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAIxE;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE;IACrD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,qBAAqB,CAAC;CACnC,GAAG,iCAAiC,CAMpC;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,cAAc,GAAG,0BAA0B,CAE/F;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,mBAAmB,CAAC;CACjC,GAAG,mBAAmB,EAAE,CAGxB;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,GAC5B,2BAA2B,CAiB7B;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,oBAAoB,GAC5B,+BAA+B,CAEjC;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAEvE"}
+{"version":3,"file":"generic-spec.d.ts","sourceRoot":"","sources":["../src/generic-spec.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,oBAAoB,EAEzB,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,+BAA+B,EACpC,KAAK,2BAA2B,EAChC,KAAK,uBAAuB,EAC7B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,KAAK,EACV,mBAAmB,EACnB,uBAAuB,EACxB,MAAM,kCAAkC,CAAC;AAG1C,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AACpD,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAEpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE;QACV,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,OAAO,CAAC,EAAE,+BAA+B,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;CACpC;AAED,MAAM,WAAW,yBAA0B,SAAQ,0BAA0B;IAC3E,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,6BAA8B,SAAQ,0BAA0B;IAC/E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC;CACnC;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,gBAAgB,CAAC;IACxB,OAAO,CAAC,EAAE,6BAA6B,CAAC;IACxC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;IAChD,OAAO,CAAC,EAAE,6BAA6B,CAAC;CACzC;AAED,MAAM,WAAW,qBAAsB,SAAQ,8BAA8B;IAC3E,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,CAAC,CAAC;IACX,SAAS,EAAE,qBAAqB,CAAC;IACjC,SAAS,EAAE,qBAAqB,CAAC;CAClC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,CAAC,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,MAAM,EAAE,0BAA0B,CAAC;KACpC,CAAC;IACF,SAAS,EAAE;QACT,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,gBAAgB,CAAC;QACxB,OAAO,CAAC,EAAE,6BAA6B,CAAC;QACxC,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;QAChD,OAAO,CAAC,EAAE;YACR,KAAK,EAAE,MAAM,EAAE,CAAC;YAChB,UAAU,CAAC,EAAE,qBAAqB,CAAC;YACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC;SACnC,CAAC;KACH,CAAC;IACF,WAAW,EAAE,oBAAoB,CAAC;IAClC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,MAAM,EAAE,0BAA0B,CAAC;IACnC,aAAa,EAAE,0BAA0B,CAAC;IAC1C,OAAO,CAAC,EAAE,0BAA0B,CAAC;IACrC,GAAG,EAAE,0BAA0B,CAAC;IAChC,SAAS,EAAE,0BAA0B,CAAC;CACvC;AAED,MAAM,MAAM,qBAAqB,GAAG,uBAAuB,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,MAAM,WAAW,iCAAiC;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,oBAAoB,CAAC;IAClC,GAAG,EAAE,0BAA0B,CAAC;CACjC;AAQD,eAAO,MAAM,2BAA2B,EAAE,2BAKzC,CAAC;AAEF,wBAAgB,mCAAmC,CACjD,MAAM,EAAE,MAAM,GACb,uBAAuB,CAmBzB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,MAAM,GACb,cAAc,CA6BhB;AAED,wBAAgB,iCAAiC,CAC/C,MAAM,EAAE,MAAM,GACb,oBAAoB,CAEtB;AAED,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,cAAc,GACnB,oBAAoB,CAStB;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,GAAG,cAAc,CAMjB;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,GAAG,uBAAuB,CAqB1B;AAED,wBAAgB,oCAAoC,CAClD,MAAM,EAAE,uBAAuB,GAC9B,MAAM,CAER;AAED,wBAAgB,gCAAgC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAI1E;AAED,wBAAgB,gCAAgC,CAAC,IAAI,EAAE;IACrD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,qBAAqB,CAAC;CACnC,GAAG,iCAAiC,CAMpC;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,cAAc,GAAG,0BAA0B,CAE/F;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE;IACnD,IAAI,EAAE,cAAc,CAAC;IACrB,UAAU,EAAE,mBAAmB,CAAC;CACjC,GAAG,mBAAmB,EAAE,CAGxB;AAED,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,mBAAmB,GAC9B,mBAAmB,EAAE,CAIvB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,oBAAoB,GAC5B,2BAA2B,CAiB7B;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,oBAAoB,GAC5B,+BAA+B,CAEjC;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAEvE"}

package/dist/generic-spec.js

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { isWorkbenchExecutionNetworkEgress, } from "@workbench-ai/workbench-contract";
 import YAML from "yaml";
 export const BENCHMARK_SPEC_FILE = "benchmark.yaml";
+export const CANDIDATE_SPEC_FILE = "candidate.yaml";
 export const DEFAULT_EXECUTION_RESOURCES = {
     cpu: 2,
     memoryGb: 4,
@@ -35,25 +36,20 @@ export function resolveWorkbenchResolvedSourceYaml(source) {
     rejectUnknownKeys(parsed, "resolved Workbench source", [
         "version",
         "benchmark",
-        "subject",
-        "optimizer",
+        "candidate",
     ], errors);
-    if (parsed.version !== 3) {
-        throw new Error("Resolved Workbench source version must be 3.");
+    if (parsed.version !== 4) {
+        throw new Error("Resolved Workbench source version must be 4.");
     }
     const benchmark = normalizeBenchmarkRecord(readRequiredRecord(parsed.benchmark, "resolved Workbench source.benchmark", errors), "benchmark.yaml", errors);
-    const subject = normalizeSubjectRecord(readRequiredRecord(parsed.subject, "resolved Workbench source.subject", errors), "resolved Workbench source.subject", errors);
-    const optimizer = parsed.optimizer === undefined
-        ? undefined
-        : normalizeOptimizerRecord(readRequiredRecord(parsed.optimizer, "resolved Workbench source.optimizer", errors), "optimizer YAML", errors);
+    const candidate = normalizeCandidateRecord(readRequiredRecord(parsed.candidate, "resolved Workbench source.candidate", errors), "resolved Workbench source.candidate", errors);
     if (errors.length > 0) {
         throw new Error(errors.join("\n"));
     }
     return genericSpecFromAuthoredBundle({
-        version: 3,
+        version: 4,
         benchmark: benchmark,
-        subject: subject,
-        ...(optimizer ? { optimizer } : {}),
+        candidate: candidate,
     });
 }
 export function engineResolveBindingForSourceYaml(source) {
@@ -72,32 +68,28 @@ export function engineResolveBindingForSpec(spec) {
 export function resolveWorkbenchSourceFiles(args) {
     return genericSpecFromAuthoredBundle(parseWorkbenchSourceFiles({
         benchmarkSource: args.benchmarkSource,
-        subjectSource: args.subjectSource,
-        optimizerSource: args.optimizerSource,
+        candidateSource: args.candidateSource,
+        runId: args.runId,
     }));
 }
 export function parseWorkbenchSourceFiles(args) {
     const errors = [];
     const benchmark = normalizeBenchmarkRecord(parseYamlRecord(args.benchmarkSource, BENCHMARK_SPEC_FILE), BENCHMARK_SPEC_FILE, errors);
-    const subject = normalizeSubjectRecord(parseYamlRecord(args.subjectSource ?? "", "subject YAML"), "subject YAML", errors);
-    const optimizer = args.optimizerSource?.trim()
-        ? normalizeOptimizerRecord(parseYamlRecord(args.optimizerSource, "optimizer YAML"), "optimizer YAML", errors)
-        : undefined;
+    const candidate = normalizeCandidateRecord(parseYamlRecord(args.candidateSource ?? "", "candidate YAML"), "candidate YAML", errors, args.runId ?? undefined);
     if (errors.length > 0) {
         throw new Error(errors.join("\n"));
     }
     return {
-        version: 3,
+        version: 4,
         benchmark: benchmark,
-        subject: subject,
-        ...(optimizer ? { optimizer } : {}),
+        candidate: candidate,
     };
 }
 export function serializeWorkbenchResolvedSourceYaml(source) {
     return YAML.stringify(source).trimEnd() + "\n";
 }
-export function isWorkbenchSubjectManifestPath(filePath) {
-    return /^subjects\/[^/]+\/subject\.ya?ml$/iu.test(filePath.replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/^(?:\.\/)+/u, ""));
+export function isWorkbenchCandidateManifestPath(filePath) {
+    return /^candidates\/[^/]+\/candidate\.ya?ml$/iu.test(filePath.replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/^(?:\.\/)+/u, ""));
 }
 export function resolveEngineCaseExecutionConfig(args) {
     return {
@@ -142,8 +134,13 @@ function genericSpecFromAuthoredBundle(source) {
     const engineRuntime = engineRuntimeFromConfig(source.benchmark.engine);
     const engineRun = cloneEngineInvocation(source.benchmark.engine);
     const engineResolve = cloneEngineInvocation(source.benchmark.engine);
+    const candidate = source.candidate;
+    const selectedRun = candidate.runs[candidate.selectedRunId];
+    if (!selectedRun) {
+        throw new Error(`Candidate run not found: ${candidate.selectedRunId}`);
+    }
     return {
-        version: 3,
+        version: 4,
         name: source.benchmark.name,
         description: source.benchmark.description,
         benchmark: {
@@ -151,33 +148,36 @@ function genericSpecFromAuthoredBundle(source) {
             description: source.benchmark.description,
             engine: cloneJson(source.benchmark.engine),
         },
-        subject: {
-            name: source.subject.name,
-            ...(source.subject.description ? { description: source.subject.description } : {}),
-            files: cloneJson(source.subject.files),
-            ...(source.subject.prepare ? { prepare: cloneJson(source.subject.prepare) } : {}),
+        candidate: {
+            name: candidate.name,
+            ...(candidate.description ? { description: candidate.description } : {}),
+            files: cloneJson(candidate.files),
+            ...(candidate.prepare ? { prepare: cloneJson(candidate.prepare) } : {}),
+            defaultRun: candidate.defaultRun ?? candidate.selectedRunId,
+            selectedRunId: candidate.selectedRunId,
+            selectedRunName: selectedRun.name,
+            runs: cloneJson(candidate.runs),
+            ...(candidate.improve
+                ? {
+                    improve: {
+                        edits: [...candidate.improve.edits],
+                        ...(candidate.improve.optimizeOn ? { optimizeOn: cloneJson(candidate.improve.optimizeOn) } : {}),
+                        ...(candidate.improve.selectBy ? { selectBy: cloneJson(candidate.improve.selectBy) } : {}),
+                    },
+                }
+                : {}),
         },
-        ...(source.optimizer
-            ? {
-                optimizer: {
-                    name: source.optimizer.name,
-                    ...(source.optimizer.description ? { description: source.optimizer.description } : {}),
-                    edits: [...source.optimizer.edits],
-                },
-            }
-            : {}),
         environment: cloneJson(engineRuntime),
         adapters: [
             ...new Set([
                 ...source.benchmark.adapters,
-                ...source.subject.adapters,
-                ...(source.optimizer?.adapters ?? []),
+                ...candidate.adapters,
             ]),
         ],
         engine: cloneJson(source.benchmark.engine),
         engineResolve: cloneJson(engineResolve),
-        ...(source.optimizer ? { improve: cloneJson(source.optimizer.improve) } : {}),
-        run: cloneJson(source.subject.run),
+        ...(candidate.improve ? { improve: clonePhaseAdapter(candidate.improve) } : {}),
+        run: clonePhaseAdapter(selectedRun),
         engineRun: cloneJson(engineRun),
     };
 }
@@ -192,7 +192,7 @@ function normalizeBenchmarkRecord(record, label, errors) {
         "adapters",
         "engine",
     ], errors);
-    requireVersionThree(record.version, label, errors);
+    requireVersionFour(record.version, label, errors);
     const name = readRequiredString(record.name, `${label}.name`, errors);
     const description = readRequiredString(record.description, `${label}.description`, errors);
     const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
@@ -202,7 +202,7 @@ function normalizeBenchmarkRecord(record, label, errors) {
     }
     return name && description && engine
         ? {
-            version: 3,
+            version: 4,
             name,
             description,
             adapters,
@@ -222,7 +222,7 @@ function normalizeEngineRuntimeConfig(engine, label, errors) {
         }
     }
 }
-function normalizeSubjectRecord(record, label, errors) {
+function normalizeCandidateRecord(record, label, errors, selectedRunId) {
     if (!record) {
         return null;
     }
@@ -233,28 +233,41 @@ function normalizeSubjectRecord(record, label, errors) {
         "files",
         "prepare",
         "adapters",
-        "run",
+        "defaultRun",
+        "runs",
+        "improve",
+        "selectedRunId",
     ], errors);
-    requireVersionThree(record.version, label, errors);
+    requireVersionFour(record.version, label, errors);
     const name = readRequiredString(record.name, `${label}.name`, errors);
     const description = readOptionalString(record.description, `${label}.description`, errors);
     const files = normalizePathRef(record.files, `${label}.files`, errors);
-    const prepare = normalizeSubjectPrepare(record.prepare, `${label}.prepare`, errors);
+    const prepare = normalizeCandidatePrepare(record.prepare, `${label}.prepare`, errors);
     const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
-    const run = normalizePhaseAdapter(record.run, `${label}.run`, errors);
-    return name && files && run
+    const runs = normalizeCandidateRuns(record.runs, `${label}.runs`, errors);
+    const defaultRun = readOptionalString(record.defaultRun, `${label}.defaultRun`, errors);
+    const embeddedSelectedRun = readOptionalString(record.selectedRunId, `${label}.selectedRunId`, errors);
+    const selected = selectedRunId ?? embeddedSelectedRun ?? defaultRun ?? Object.keys(runs).sort()[0];
+    if (selected && !runs[selected]) {
+        errors.push(`${label}.selectedRunId references unknown run ${selected}.`);
+    }
+    const improve = normalizeCandidateImprove(record.improve, `${label}.improve`, errors);
+    return name && files && selected && Object.keys(runs).length > 0
         ? {
-            version: 3,
+            version: 4,
             name,
             ...(description ? { description } : {}),
             files,
             ...(prepare ? { prepare } : {}),
             adapters,
-            run,
+            ...(defaultRun ? { defaultRun } : {}),
+            runs,
+            ...(improve ? { improve } : {}),
+            selectedRunId: selected,
         }
         : null;
 }
-function normalizeSubjectPrepare(value, label, errors) {
+function normalizeCandidatePrepare(value, label, errors) {
     if (value === undefined) {
         return undefined;
     }
@@ -266,38 +279,115 @@ function normalizeSubjectPrepare(value, label, errors) {
     const command = readRequiredString(record.command, `${label}.command`, errors);
     return command ? { command } : undefined;
 }
-function normalizeOptimizerRecord(record, label, errors) {
+function normalizeCandidateRuns(value, label, errors) {
+    const record = readRequiredRecord(value, label, errors);
     if (!record) {
-        return null;
+        return {};
     }
-    rejectUnknownKeys(record, label, [
-        "version",
-        "name",
-        "description",
-        "edits",
-        "adapters",
-        "improve",
-    ], errors);
-    requireVersionThree(record.version, label, errors);
-    const name = readRequiredString(record.name, `${label}.name`, errors);
-    const description = readOptionalString(record.description, `${label}.description`, errors);
+    const runs = {};
+    for (const [runId, runValue] of Object.entries(record).sort(([left], [right]) => left.localeCompare(right))) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/u.test(runId)) {
+            errors.push(`${label}.${runId} must use letters, numbers, dots, underscores, or dashes.`);
+            continue;
+        }
+        const runRecord = readRequiredRecord(runValue, `${label}.${runId}`, errors);
+        if (!runRecord) {
+            continue;
+        }
+        rejectUnknownKeys(runRecord, `${label}.${runId}`, ["name", "use", "with", "auth"], errors);
+        const name = readRequiredString(runRecord.name, `${label}.${runId}.name`, errors);
+        const invocation = normalizePhaseAdapter(adapterRecordFrom(runRecord), `${label}.${runId}`, errors);
+        if (name && invocation) {
+            runs[runId] = {
+                name,
+                ...invocation,
+            };
+        }
+    }
+    if (Object.keys(runs).length === 0) {
+        errors.push(`${label} must declare at least one run.`);
+    }
+    return runs;
+}
+function normalizeCandidateImprove(value, label, errors) {
+    if (value === undefined) {
+        return undefined;
+    }
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return undefined;
+    }
+    rejectUnknownKeys(record, label, ["edits", "use", "with", "auth", "optimizeOn", "selectBy"], errors);
     const edits = normalizeRelativePathList(record.edits, `${label}.edits`, errors);
-    const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
-    const improve = normalizePhaseAdapter(record.improve, `${label}.improve`, errors);
-    return name && edits.length > 0 && improve
+    const invocation = normalizePhaseAdapter(adapterRecordFrom(record), label, errors);
+    const optimizeOn = normalizeCaseSelector(record.optimizeOn, `${label}.optimizeOn`, errors);
+    const selectBy = normalizeSelectionSpec(record.selectBy, `${label}.selectBy`, errors);
+    return edits.length > 0 && invocation
         ? {
-            version: 3,
-            name,
-            ...(description ? { description } : {}),
+            ...invocation,
             edits,
-            adapters,
-            improve,
+            ...(optimizeOn ? { optimizeOn } : {}),
+            ...(selectBy ? { selectBy } : {}),
         }
-        : null;
+        : undefined;
+}
+function normalizeSelectionSpec(value, label, errors) {
+    if (value === undefined) {
+        return undefined;
+    }
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return undefined;
+    }
+    rejectUnknownKeys(record, label, ["metric", "cases"], errors);
+    const metric = readRequiredString(record.metric, `${label}.metric`, errors);
+    const cases = normalizeCaseSelector(record.cases, `${label}.cases`, errors);
+    return metric
+        ? {
+            metric,
+            ...(cases ? { cases } : {}),
+        }
+        : undefined;
+}
+function normalizeCaseSelector(value, label, errors) {
+    if (value === undefined) {
+        return undefined;
+    }
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return undefined;
+    }
+    rejectUnknownKeys(record, label, ["all", "split"], errors);
+    const hasAll = Object.prototype.hasOwnProperty.call(record, "all");
+    const hasSplit = Object.prototype.hasOwnProperty.call(record, "split");
+    if (hasAll && hasSplit) {
+        errors.push(`${label} must specify either all or split, not both.`);
+        return undefined;
+    }
+    if (!hasAll && !hasSplit) {
+        errors.push(`${label} must specify all: true or split.`);
+        return undefined;
+    }
+    if (hasAll) {
+        if (record.all !== true) {
+            errors.push(`${label}.all must be true when provided.`);
+            return undefined;
+        }
+        return { all: true };
+    }
+    const split = readRequiredString(record.split, `${label}.split`, errors);
+    return split ? { split } : undefined;
 }
-function requireVersionThree(value, label, errors) {
-    if (value !== 3) {
-        errors.push(`${label}.version must be 3.`);
+function adapterRecordFrom(record) {
+    return {
+        use: record.use,
+        ...(record.with !== undefined ? { with: record.with } : {}),
+        ...(record.auth !== undefined ? { auth: record.auth } : {}),
+    };
+}
+function requireVersionFour(value, label, errors) {
+    if (value !== 4) {
+        errors.push(`${label}.version must be 4.`);
     }
 }
 function normalizeRuntime(value, label, errors) {
@@ -417,10 +507,13 @@ function engineRuntimeFromConfig(engine) {
     };
 }
 function cloneEngineInvocation(engine) {
+    return clonePhaseAdapter(engine);
+}
+function clonePhaseAdapter(adapter) {
     return {
-        use: engine.use,
-        with: cloneJson(engine.with ?? {}),
-        ...(engine.auth !== undefined ? { auth: cloneJson(engine.auth) } : {}),
+        use: adapter.use,
+        with: cloneJson(adapter.with ?? {}),
+        ...(adapter.auth !== undefined ? { auth: cloneJson(adapter.auth) } : {}),
     };
 }
 function mergeRuntime(base, override) {