@workbench-ai/workbench 0.0.49 → 0.0.51

package/dist/project-source.js

@@ -2,16 +2,15 @@ import { promises as fs } from "node:fs";
 import { createHash, randomUUID } from "node:crypto";
 import { spawn } from "node:child_process";
 import path from "node:path";
-import { BENCHMARK_SPEC_FILE, buildWorkbenchProjectSourceFiles, engineResolveInvocationForSpec, normalizeSurfaceFiles, parseWorkbenchSourceFiles, resolveWorkbenchResolvedSourceYaml, serializeWorkbenchResolvedSourceYaml, validateWorkbenchResolvedSourceYaml, } from "@workbench-ai/workbench-core";
+import { BENCHMARK_SPEC_FILE, CANDIDATE_SPEC_FILE, buildWorkbenchProjectSourceFiles, engineResolveInvocationForSpec, normalizeSurfaceFiles, parseWorkbenchSourceFiles, resolveWorkbenchResolvedSourceYaml, serializeWorkbenchResolvedSourceYaml, validateWorkbenchResolvedSourceYaml, } from "@workbench-ai/workbench-core";
 import { assertWorkbenchAdapterOperationSupport, assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterInvocations, readWorkbenchAdapterOperationResult, workbenchAdapterOperationCommand, workbenchAdapterOperationResultPath, } from "@workbench-ai/workbench-protocol";
 import { readSnapshotFiles, WorkspaceSnapshotError, } from "./workspace-snapshot.js";
 import { defaultAdapterManifests, composeRuntimeDockerfileWithAdapters, resolveDefaultWorkbenchAdapter, resolveProjectAdapterSource, resolveWorkbenchAdaptersForProject, } from "./adapter-project.js";
 import { createAdapterCommandEnv } from "./adapter-command-env.js";
 import YAML from "yaml";
 export const WORKBENCH_BENCHMARK_FILE = BENCHMARK_SPEC_FILE;
-export const WORKBENCH_SUBJECTS_DIR = "subjects";
-export const WORKBENCH_OPTIMIZERS_DIR = "optimizers";
-export const WORKBENCH_SUBJECT_FILE = "subject.yaml";
+export const WORKBENCH_CANDIDATES_DIR = "candidates";
+export const WORKBENCH_CANDIDATE_FILE = CANDIDATE_SPEC_FILE;
 function rootAdapterInvocations(spec) {
     return [
         engineResolveInvocationForSpec(spec),
@@ -22,25 +21,20 @@ function rootAdapterInvocations(spec) {
 }
 export async function readLocalProjectSource(source, options = {}) {
     const paths = await resolveLocalProjectSourcePaths(source, options);
-    const { dir, benchmarkPath, subjectSpecPath, subjectDir, optimizerPath, } = paths;
+    const { dir, benchmarkPath, candidateSpecPath, candidateDir, } = paths;
     const benchmarkSource = await readRequiredTextFile(benchmarkPath, WORKBENCH_BENCHMARK_FILE);
-    const subjectSource = await readRequiredTextFile(subjectSpecPath, "subject YAML");
-    const optimizerSource = optimizerPath
-        ? await readRequiredTextFile(optimizerPath, "optimizer YAML")
-        : undefined;
+    const candidateSource = await readRequiredTextFile(candidateSpecPath, "candidate YAML");
     const normalizedSources = await normalizeSourceYamlForExecution({
         dir,
         benchmarkPath,
         benchmarkSource,
-        subjectSpecPath,
-        subjectSource,
-        optimizerPath,
-        optimizerSource,
+        candidateSpecPath,
+        candidateSource,
     });
     const resolvedSource = parseWorkbenchSourceFiles({
         benchmarkSource: normalizedSources.benchmarkSource,
-        subjectSource: normalizedSources.subjectSource,
-        optimizerSource: normalizedSources.optimizerSource,
+        candidateSource: normalizedSources.candidateSource,
+        runId: options.runId,
     });
     const specSource = serializeWorkbenchResolvedSourceYaml(resolvedSource);
     const validation = validateWorkbenchResolvedSourceYaml(specSource);
@@ -60,10 +54,10 @@ export async function readLocalProjectSource(source, options = {}) {
     ];
     const composedDockerfile = await composeRuntimeDockerfileWithAdapters(dockerfile, adapters);
     const adapterFiles = adapterSourceFiles(adapters);
-    const absoluteSubjectFilesPath = resolveProjectPath(dir, spec.subject.files.path);
-    const subjectFilesPath = absoluteSubjectFilesPath;
-    const subjectFiles = await directoryExists(absoluteSubjectFilesPath)
-        ? normalizeSurfaceFiles(await readSnapshotFiles(absoluteSubjectFilesPath))
+    const absoluteCandidateFilesPath = resolveProjectPath(dir, spec.candidate.files.path);
+    const candidateFilesPath = absoluteCandidateFilesPath;
+    const candidateFiles = await directoryExists(absoluteCandidateFilesPath)
+        ? normalizeSurfaceFiles(await readSnapshotFiles(absoluteCandidateFilesPath))
         : [];
     const rawEngineResolveFiles = engineResolveFilesFromBundles(normalizedSources.engineCases);
     const engineResolveFiles = toHostedFiles(rawEngineResolveFiles);
@@ -79,19 +73,20 @@ export async function readLocalProjectSource(source, options = {}) {
         spec,
         benchmarkPath,
         benchmarkSource,
-        subjectName: path.basename(subjectDir),
-        subjectDir,
-        subjectFilesPath,
-        subjectSpecPath,
-        subjectSource,
-        ...(optimizerSource !== undefined && optimizerPath ? { optimizerPath, optimizerSource } : {}),
+        candidateName: path.basename(candidateDir),
+        candidateDir,
+        candidateFilesPath,
+        candidateSpecPath,
+        candidateSource,
+        candidateRunId: spec.candidate.selectedRunId,
+        candidateRunIds: Object.keys(spec.candidate.runs).sort(),
         benchmarkAdapterSources: [...resolvedSource.benchmark.adapters],
         benchmarkAdapterIds,
         dockerfilePath,
         dockerfile,
         runtimeDockerfile: composedDockerfile,
         dockerfileFiles: toHostedFiles(dockerfileSourceFiles(dockerfileSources)),
-        subjectFiles: toHostedFiles(subjectFiles),
+        candidateFiles: toHostedFiles(candidateFiles),
         engineResolveFiles,
         adapters,
         adapterFiles: toHostedFiles(adapterFiles),
@@ -105,13 +100,10 @@ export async function readLocalProjectSource(source, options = {}) {
         sourceFiles: buildWorkbenchProjectSourceFiles({
             specFiles: [
                 textSourceFile(toRootRelativePath(dir, benchmarkPath), benchmarkSource),
-                textSourceFile(toRootRelativePath(dir, subjectSpecPath), subjectSource),
-                ...(optimizerSource !== undefined && optimizerPath
-                    ? [textSourceFile(toRootRelativePath(dir, optimizerPath), optimizerSource)]
-                    : []),
+                textSourceFile(toRootRelativePath(dir, candidateSpecPath), candidateSource),
             ],
-            subjectFilesPath: spec.subject.files.path,
-            subjectFiles,
+            candidateFilesPath: spec.candidate.files.path,
+            candidateFiles: candidateFiles,
             engineResolveFilesPath: normalizedSources.engineResolveFingerprintPath,
             engineResolveFiles: rawEngineResolveFiles,
             adapterFiles,
@@ -120,16 +112,13 @@ export async function readLocalProjectSource(source, options = {}) {
     };
 }
 export async function readLocalAuthoredProjectSource(source, options = {}) {
-    const { dir, benchmarkPath, subjectSpecPath, subjectDir, optimizerPath, } = await resolveLocalProjectSourcePaths(source, options);
+    const { dir, benchmarkPath, candidateSpecPath, candidateDir, } = await resolveLocalProjectSourcePaths(source, options);
     const benchmarkSource = await readRequiredTextFile(benchmarkPath, WORKBENCH_BENCHMARK_FILE);
-    const subjectSource = await readRequiredTextFile(subjectSpecPath, "subject YAML");
-    const optimizerSource = optimizerPath
-        ? await readRequiredTextFile(optimizerPath, "optimizer YAML")
-        : undefined;
+    const candidateSource = await readRequiredTextFile(candidateSpecPath, "candidate YAML");
     const resolvedSource = parseWorkbenchSourceFiles({
         benchmarkSource,
-        subjectSource,
-        optimizerSource,
+        candidateSource,
+        runId: options.runId,
     });
     const specSource = serializeWorkbenchResolvedSourceYaml(resolvedSource);
     return {
@@ -138,16 +127,12 @@ export async function readLocalAuthoredProjectSource(source, options = {}) {
         specSource,
         benchmarkPath,
         benchmarkSource,
-        subjectDir,
-        subjectSpecPath,
-        subjectSource,
-        ...(optimizerPath && optimizerSource !== undefined ? { optimizerPath, optimizerSource } : {}),
+        candidateDir,
+        candidateSpecPath,
+        candidateSource,
         sourceFiles: [
             textSourceFile(toRootRelativePath(dir, benchmarkPath), benchmarkSource),
-            textSourceFile(toRootRelativePath(dir, subjectSpecPath), subjectSource),
-            ...(optimizerPath && optimizerSource !== undefined
-                ? [textSourceFile(toRootRelativePath(dir, optimizerPath), optimizerSource)]
-                : []),
+            textSourceFile(toRootRelativePath(dir, candidateSpecPath), candidateSource),
         ],
     };
 }
@@ -156,105 +141,67 @@ async function resolveLocalProjectSourcePaths(source, options) {
     const stat = await fs.stat(resolved).catch(() => null);
     if (stat?.isFile()) {
         const sourceRecord = await readYamlRecordFile(resolved);
-        if (isSubjectSourceRecord(sourceRecord)) {
-            const subjectDir = path.dirname(resolved);
-            const dir = projectRootForSubjectDir(subjectDir);
+        if (isCandidateSourceRecord(sourceRecord)) {
+            const candidateDir = path.dirname(resolved);
+            const dir = projectRootForCandidateDir(candidateDir);
             return {
                 dir,
                 benchmarkPath: path.join(dir, WORKBENCH_BENCHMARK_FILE),
-                subjectDir,
-                subjectSpecPath: resolved,
-                optimizerPath: await resolveOptimizerPath(dir, options.optimizerPath, path.basename(subjectDir)),
+                candidateDir,
+                candidateSpecPath: resolved,
             };
         }
         if (isBenchmarkSourceRecord(sourceRecord)) {
             const dir = path.dirname(resolved);
-            const subjectPaths = await resolveSubjectPaths(dir);
+            const candidatePaths = await resolveCandidatePaths(dir);
             return {
                 dir,
                 benchmarkPath: resolved,
-                ...subjectPaths,
-                optimizerPath: await resolveOptimizerPath(dir, options.optimizerPath, path.basename(subjectPaths.subjectDir)),
+                ...candidatePaths,
             };
         }
-        if (isOptimizerSourceRecord(sourceRecord)) {
-            throw new WorkspaceSnapshotError(`Optimizer source must be passed with --optimizer; pass a source directory or subject YAML as SOURCE: ${resolved}`);
-        }
         throw new WorkspaceSnapshotError(`Unsupported Workbench YAML source: ${resolved}`);
     }
     const dir = resolved;
-    const directorySubject = await subjectPathsForSubjectDirectory(dir);
-    if (directorySubject) {
-        return {
-            ...directorySubject,
-            optimizerPath: await resolveOptimizerPath(directorySubject.dir, options.optimizerPath, path.basename(directorySubject.subjectDir)),
-        };
+    const directoryCandidate = await candidatePathsForCandidateDirectory(dir);
+    if (directoryCandidate) {
+        return directoryCandidate;
     }
-    const subjectPaths = await resolveSubjectPathsWithOptimizer(dir, options.optimizerPath);
+    const candidatePaths = await resolveCandidatePaths(dir);
     return {
         dir,
         benchmarkPath: path.join(dir, WORKBENCH_BENCHMARK_FILE),
-        ...subjectPaths,
-    };
-}
-async function resolveSubjectPathsWithOptimizer(dir, explicitOptimizerPath) {
-    const subjectPaths = await resolveSubjectPaths(dir);
-    return {
-        ...subjectPaths,
-        optimizerPath: await resolveOptimizerPath(dir, explicitOptimizerPath, path.basename(subjectPaths.subjectDir)),
+        ...candidatePaths,
     };
 }
-async function resolveSubjectPaths(dir) {
-    const subjectsDir = path.join(dir, WORKBENCH_SUBJECTS_DIR);
-    const subjects = await listSubjectManifestFiles(subjectsDir);
-    if (subjects.length === 1) {
-        const subjectSpecPath = subjects[0];
-        const subjectDir = path.dirname(subjectSpecPath);
+async function resolveCandidatePaths(dir) {
+    const candidatesDir = path.join(dir, WORKBENCH_CANDIDATES_DIR);
+    const candidates = await listCandidateManifestFiles(candidatesDir);
+    if (candidates.length === 1) {
+        const candidateSpecPath = candidates[0];
+        const candidateDir = path.dirname(candidateSpecPath);
         return {
-            subjectDir,
-            subjectSpecPath,
+            candidateDir,
+            candidateSpecPath,
         };
     }
-    if (subjects.length > 1) {
-        throw new WorkspaceSnapshotError(`Multiple subject directories found under ${subjectsDir}; pass subjects/NAME or subjects/NAME/subject.yaml explicitly.`);
-    }
-    throw new WorkspaceSnapshotError(`No subject directories found under ${subjectsDir}; create subjects/NAME/subject.yaml with files.path.`);
-}
-async function resolveOptimizerPath(dir, explicit, subjectName) {
-    if (explicit) {
-        return path.resolve(dir, explicit);
-    }
-    if (subjectName) {
-        const named = path.join(dir, WORKBENCH_OPTIMIZERS_DIR, `${subjectName}.yaml`);
-        if (await fileExists(named)) {
-            return named;
-        }
-    }
-    const optimizersDir = path.join(dir, WORKBENCH_OPTIMIZERS_DIR);
-    const optimizers = await listYamlFiles(optimizersDir);
-    if (optimizers.length === 1) {
-        return optimizers[0];
+    if (candidates.length > 1) {
+        throw new WorkspaceSnapshotError(`Multiple candidate directories found under ${candidatesDir}; pass candidates/NAME or candidates/NAME/candidate.yaml explicitly.`);
     }
-    return undefined;
+    throw new WorkspaceSnapshotError(`No candidate directories found under ${candidatesDir}; create candidates/NAME/candidate.yaml with files.path.`);
 }
 async function normalizeSourceYamlForExecution(args) {
     const benchmark = parseYamlRecord(args.benchmarkSource, args.benchmarkPath);
-    const subject = parseYamlRecord(args.subjectSource, args.subjectSpecPath);
-    const optimizer = args.optimizerSource === undefined || args.optimizerPath === undefined
-        ? undefined
-        : parseYamlRecord(args.optimizerSource, args.optimizerPath);
+    const candidate = parseYamlRecord(args.candidateSource, args.candidateSpecPath);
     const benchmarkDir = path.dirname(args.benchmarkPath);
-    const subjectDir = path.dirname(args.subjectSpecPath);
+    const candidateDir = path.dirname(args.candidateSpecPath);
     normalizeAdapterSourcePaths(args.dir, benchmark, benchmarkDir);
-    normalizeAdapterSourcePaths(args.dir, subject, subjectDir);
-    if (optimizer && args.optimizerPath) {
-        normalizeAdapterSourcePaths(args.dir, optimizer, path.dirname(args.optimizerPath));
-    }
+    normalizeAdapterSourcePaths(args.dir, candidate, candidateDir);
     const engine = yamlRecord(benchmark.engine);
     if (!engine || typeof engine.use !== "string" || !engine.use.trim()) {
         throw new WorkspaceSnapshotError("benchmark.yaml engine must declare an adapter invocation with use.");
     }
-    normalizeEngineForExecution(args.dir, benchmarkDir, subjectDir, benchmark, subject);
+    normalizeEngineForExecution(args.dir, benchmarkDir, candidateDir, benchmark, candidate);
     const authoredEngineResolve = engineResolveInvocationFromRecord(engine);
     const engineResolve = await resolveEngineResolveAdapter({
         root: args.dir,
@@ -267,21 +214,18 @@ async function normalizeSourceYamlForExecution(args) {
     applyEngineResolveEnvironment(benchmark, engineResolve.environment);
     return {
         benchmarkSource: YAML.stringify(benchmark).trimEnd() + "\n",
-        subjectSource: YAML.stringify(subject).trimEnd() + "\n",
+        candidateSource: YAML.stringify(candidate).trimEnd() + "\n",
         engineResolveFingerprintPath,
         engineResolve: authoredEngineResolve,
         ...(engineResolve.environment ? { engineResolveEnvironment: engineResolve.environment } : {}),
         engineCases: engineResolve.engineCases,
-        ...(optimizer
-            ? { optimizerSource: YAML.stringify(optimizer).trimEnd() + "\n" }
-            : {}),
     };
 }
-function normalizeEngineForExecution(root, benchmarkDir, subjectDir, benchmark, subject) {
-    const subjectFiles = yamlRecord(subject.files);
-    if (subjectFiles && typeof subjectFiles.path === "string") {
-        subjectFiles.path = toRootRelativePath(root, resolveYamlReference(subjectDir, subjectFiles.path));
-        subject.files = subjectFiles;
+function normalizeEngineForExecution(root, benchmarkDir, candidateDir, benchmark, candidate) {
+    const candidateFiles = yamlRecord(candidate.files);
+    if (candidateFiles && typeof candidateFiles.path === "string") {
+        candidateFiles.path = toRootRelativePath(root, resolveYamlReference(candidateDir, candidateFiles.path));
+        candidate.files = candidateFiles;
     }
     const engine = yamlRecord(benchmark.engine);
     const engineConfig = yamlRecord(engine?.with) ?? {};
@@ -569,34 +513,31 @@ function isPathAdapterSource(source) {
 function isBenchmarkSourceRecord(record) {
     return record.engine !== undefined;
 }
-function isSubjectSourceRecord(record) {
-    return record.run !== undefined;
-}
-function isOptimizerSourceRecord(record) {
-    return record.edits !== undefined && record.improve !== undefined;
+function isCandidateSourceRecord(record) {
+    return record.runs !== undefined;
 }
 async function readYamlRecordFile(filePath) {
     return parseYamlRecord(await readRequiredTextFile(filePath, path.basename(filePath)), filePath);
 }
-async function subjectPathsForSubjectDirectory(sourceDir) {
-    const subjectSpecPath = path.join(sourceDir, WORKBENCH_SUBJECT_FILE);
-    if (!(await fileExists(subjectSpecPath))) {
+async function candidatePathsForCandidateDirectory(sourceDir) {
+    const candidateSpecPath = path.join(sourceDir, WORKBENCH_CANDIDATE_FILE);
+    if (!(await fileExists(candidateSpecPath))) {
         return null;
     }
-    const dir = projectRootForSubjectDir(sourceDir);
+    const dir = projectRootForCandidateDir(sourceDir);
     return {
         dir,
         benchmarkPath: path.join(dir, WORKBENCH_BENCHMARK_FILE),
-        subjectDir: sourceDir,
-        subjectSpecPath,
+        candidateDir: sourceDir,
+        candidateSpecPath,
     };
 }
-function projectRootForSubjectDir(subjectDir) {
-    const parent = path.basename(path.dirname(subjectDir));
-    if (parent !== WORKBENCH_SUBJECTS_DIR) {
-        throw new WorkspaceSnapshotError(`Subject directory must be under ${WORKBENCH_SUBJECTS_DIR}/NAME: ${subjectDir}`);
+function projectRootForCandidateDir(candidateDir) {
+    const parent = path.basename(path.dirname(candidateDir));
+    if (parent !== WORKBENCH_CANDIDATES_DIR) {
+        throw new WorkspaceSnapshotError(`Candidate directory must be under ${WORKBENCH_CANDIDATES_DIR}/NAME: ${candidateDir}`);
     }
-    return path.dirname(path.dirname(subjectDir));
+    return path.dirname(path.dirname(candidateDir));
 }
 function parseYamlRecord(source, label) {
     const parsed = YAML.parse(source);
@@ -632,7 +573,7 @@ async function fileExists(filePath) {
 async function directoryExists(filePath) {
     return await fs.stat(filePath).then((stat) => stat.isDirectory(), () => false);
 }
-async function listSubjectManifestFiles(dir) {
+async function listCandidateManifestFiles(dir) {
     const entries = await fs.readdir(dir, { withFileTypes: true }).catch((error) => {
         if (error.code === "ENOENT") {
             return [];
@@ -642,25 +583,13 @@ async function listSubjectManifestFiles(dir) {
     const manifests = await Promise.all(entries
         .filter((entry) => entry.isDirectory())
         .map(async (entry) => {
-        const manifestPath = path.join(dir, entry.name, WORKBENCH_SUBJECT_FILE);
+        const manifestPath = path.join(dir, entry.name, WORKBENCH_CANDIDATE_FILE);
         return await fileExists(manifestPath) ? manifestPath : null;
     }));
     return manifests
         .filter((entry) => Boolean(entry))
         .sort((left, right) => left.localeCompare(right));
 }
-async function listYamlFiles(dir) {
-    const entries = await fs.readdir(dir, { withFileTypes: true }).catch((error) => {
-        if (error.code === "ENOENT") {
-            return [];
-        }
-        throw error;
-    });
-    return entries
-        .filter((entry) => entry.isFile() && /\.ya?ml$/iu.test(entry.name))
-        .map((entry) => path.join(dir, entry.name))
-        .sort((left, right) => left.localeCompare(right));
-}
 async function readRequiredTextFile(filePath, label) {
     return await fs.readFile(filePath, "utf8").catch((error) => {
         if (error.code === "ENOENT") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workbench-ai/workbench",
-  "version": "0.0.49",
+  "version": "0.0.51",
   "type": "module",
   "repository": {
     "type": "git",
@@ -21,9 +21,9 @@
   ],
   "dependencies": {
     "yaml": "^2.8.2",
-    "@workbench-ai/workbench-built-in-adapters": "0.0.49",
-    "@workbench-ai/workbench-protocol": "0.0.49",
-    "@workbench-ai/workbench-core": "0.0.49"
+    "@workbench-ai/workbench-core": "0.0.51",
+    "@workbench-ai/workbench-protocol": "0.0.51",
+    "@workbench-ai/workbench-built-in-adapters": "0.0.51"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.2.2",