@workbench-ai/workbench 0.0.49 → 0.0.51

package/dist/local-archive.js

@@ -1,36 +1,37 @@
 import { promises as fs } from "node:fs";
 import path from "node:path";
-import { buildWorkbenchTraceSessionsFromFiles, selectExecutionOutputFilesForInspection, } from "@workbench-ai/workbench-core";
+import { buildWorkbenchTraceSessionsFromFiles, candidateRecordWithoutDerivedFields, sanitizeWorkbenchRuntimeCandidateForExchange, sanitizeWorkbenchRuntimeJobForExchange, selectExecutionOutputFilesForInspection, workbenchRuntimeBundleStats, workbenchSurfaceFilesEqualForExchange, } from "@workbench-ai/workbench-core";
 const RUNTIME_DIR = ".workbench/runtime";
+const CANDIDATE_RECORDS_DIR = "candidates";
 export function localRuntimeDir(workspace) {
     return path.join(workspace, RUNTIME_DIR);
 }
 export async function loadLocalArchive(workspace) {
     const index = await loadLocalArchiveIndex(workspace);
     const root = localRuntimeDir(workspace);
-    const subjectFiles = {};
-    await Promise.all(index.subjects.map(async (subject) => {
-        subjectFiles[subject.id] = await readSurfaceFiles(path.join(root, "subjects", localRecordName(subject.id), "files"));
+    const candidateFiles = {};
+    await Promise.all(index.candidates.map(async (candidate) => {
+        candidateFiles[candidate.id] = await readSurfaceFiles(path.join(root, CANDIDATE_RECORDS_DIR, localRecordName(candidate.id), "files"));
     }));
     const snapshot = {
         ...index,
-        subjectFiles,
+        candidateFiles,
     };
     validateLocalArchiveSnapshot(snapshot);
     return snapshot;
 }
 export async function loadLocalArchiveIndex(workspace) {
     const root = localRuntimeDir(workspace);
-    const [state, subjects, evaluations, runs, events] = await Promise.all([
+    const [state, candidates, evaluations, runs, events] = await Promise.all([
         readJson(path.join(root, "state.json"), {}),
-        readRecords(path.join(root, "subjects"), "record.json"),
+        readRecords(path.join(root, CANDIDATE_RECORDS_DIR), "record.json"),
         readFlatRecords(path.join(root, "evaluations")),
         readFlatRecords(path.join(root, "runs")),
         readJson(path.join(root, "events.json"), []),
     ]);
     const index = {
         activeId: typeof state.activeId === "string" ? state.activeId : null,
-        subjects: subjects.sort((left, right) => left.ordinal - right.ordinal || left.id.localeCompare(right.id)),
+        candidates: candidates.sort(compareLocalCandidateRecords),
         evaluations: evaluations.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id)),
         runs: runs.sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id)),
         events: events.sort((left, right) => left.at.localeCompare(right.at) || left.id.localeCompare(right.id)),
@@ -42,19 +43,19 @@ export async function saveLocalArchive(workspace, snapshot) {
     const root = localRuntimeDir(workspace);
     await fs.mkdir(root, { recursive: true });
     await writeJson(path.join(root, "state.json"), { activeId: snapshot.activeId });
-    await fs.rm(path.join(root, "subjects"), { force: true, recursive: true });
+    await fs.rm(path.join(root, CANDIDATE_RECORDS_DIR), { force: true, recursive: true });
     await fs.rm(path.join(root, "evaluations"), { force: true, recursive: true });
     await fs.rm(path.join(root, "runs"), { force: true, recursive: true });
     await Promise.all([
-        fs.mkdir(path.join(root, "subjects"), { recursive: true }),
+        fs.mkdir(path.join(root, CANDIDATE_RECORDS_DIR), { recursive: true }),
         fs.mkdir(path.join(root, "evaluations"), { recursive: true }),
         fs.mkdir(path.join(root, "runs"), { recursive: true }),
     ]);
-    for (const subject of snapshot.subjects) {
-        const subjectRoot = path.join(root, "subjects", subject.id);
-        await fs.mkdir(subjectRoot, { recursive: true });
-        await writeJson(path.join(subjectRoot, "record.json"), subject);
-        await writeSurfaceFiles(path.join(subjectRoot, "files"), snapshot.subjectFiles[subject.id] ?? []);
+    for (const candidate of snapshot.candidates) {
+        const candidateRoot = path.join(root, CANDIDATE_RECORDS_DIR, candidate.id);
+        await fs.mkdir(candidateRoot, { recursive: true });
+        await writeJson(path.join(candidateRoot, "record.json"), candidateRecordWithoutDerivedFields(candidate));
+        await writeSurfaceFiles(path.join(candidateRoot, "files"), snapshot.candidateFiles[candidate.id] ?? []);
     }
     for (const evaluation of snapshot.evaluations) {
         await writeJson(path.join(root, "evaluations", `${evaluation.id}.json`), evaluation);
@@ -65,6 +66,142 @@ export async function saveLocalArchive(workspace, snapshot) {
     await writeJson(path.join(root, "events.json"), snapshot.events);
 }
 export async function saveLocalJobs(workspace, jobs) {
+    if (jobs.length === 0) {
+        return;
+    }
+    await writeArchivedLocalJobs(workspace, jobs, new Map());
+}
+export async function exportLocalRuntimeBundle(workspace) {
+    const snapshot = await loadLocalArchive(workspace);
+    const jobs = (await readLocalJobs(workspace)).map(sanitizeRuntimeJobForExchange);
+    const executionFiles = await Promise.all(jobs.map(async (job) => ({
+        jobId: job.id,
+        files: await readLocalExecutionFiles(workspace, job.id),
+    })));
+    return {
+        schema: "workbench.runtime.bundle.v1",
+        activeId: snapshot.activeId,
+        candidates: snapshot.candidates.map(sanitizeWorkbenchRuntimeCandidateForExchange),
+        candidateFiles: Object.entries(snapshot.candidateFiles).map(([candidateId, files]) => ({
+            candidateId,
+            files: copySurfaceFiles(files),
+        })),
+        evaluations: snapshot.evaluations.map((evaluation) => ({ ...evaluation })),
+        runs: snapshot.runs.map((run) => ({ ...run })),
+        jobs,
+        executionFiles,
+        events: snapshot.events.map((event) => ({ ...event })),
+    };
+}
+export async function importLocalRuntimeBundle(workspace, bundle) {
+    validateRuntimeBundleSchema(bundle);
+    const snapshot = await loadLocalArchive(workspace);
+    const existingJobs = (await readLocalJobs(workspace)).map(sanitizeRuntimeJobForExchange);
+    let changed = false;
+    const existingCandidates = snapshot.candidates.map(sanitizeWorkbenchRuntimeCandidateForExchange);
+    if (JSON.stringify(existingCandidates) !== JSON.stringify(snapshot.candidates)) {
+        changed = true;
+    }
+    const incomingCandidates = bundle.candidates.map(sanitizeWorkbenchRuntimeCandidateForExchange);
+    const candidates = mergeRecordsById(existingCandidates, incomingCandidates, (candidate) => candidate.id, (didChange) => {
+        changed ||= didChange;
+    }).sort(compareLocalCandidateRecords);
+    const candidateFiles = { ...snapshot.candidateFiles };
+    for (const group of bundle.candidateFiles) {
+        const candidateId = localRecordName(group.candidateId);
+        const files = copySurfaceFiles(group.files);
+        const existing = candidateFiles[candidateId];
+        if (existing) {
+            if (!workbenchSurfaceFilesEqualForExchange(existing, files)) {
+                throw new Error(`Runtime history conflict for candidate files ${candidateId}.`);
+            }
+        }
+        else {
+            changed = true;
+        }
+        candidateFiles[candidateId] = files;
+    }
+    const candidateIds = new Set(candidates.map((candidate) => candidate.id));
+    const activeId = bundle.activeId && candidateIds.has(bundle.activeId)
+        ? bundle.activeId
+        : snapshot.activeId && candidateIds.has(snapshot.activeId)
+            ? snapshot.activeId
+            : null;
+    if (activeId !== snapshot.activeId) {
+        changed = true;
+    }
+    const evaluations = mergeRecordsById(snapshot.evaluations, bundle.evaluations, (evaluation) => evaluation.id, (didChange) => {
+        changed ||= didChange;
+    }).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
+    const runs = mergeRecordsById(snapshot.runs, bundle.runs, (run) => run.id, (didChange) => {
+        changed ||= didChange;
+    }).sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
+    const events = mergeRecordsById(snapshot.events, bundle.events, runtimeEventKey, (didChange) => {
+        changed ||= didChange;
+    }).sort((left, right) => left.at.localeCompare(right.at) || left.id.localeCompare(right.id));
+    const executionFilesByJobId = new Map();
+    await Promise.all(existingJobs.map(async (job) => {
+        executionFilesByJobId.set(job.id, await readLocalExecutionFiles(workspace, job.id));
+    }));
+    for (const group of bundle.executionFiles) {
+        const jobId = localRecordName(group.jobId);
+        const files = copySurfaceFiles(group.files);
+        const existing = executionFilesByJobId.get(jobId);
+        if (existing) {
+            if (!workbenchSurfaceFilesEqualForExchange(existing, files)) {
+                throw new Error(`Runtime history conflict for execution files ${jobId}.`);
+            }
+        }
+        else {
+            changed = true;
+        }
+        executionFilesByJobId.set(jobId, files);
+    }
+    const jobs = mergeRecordsById(existingJobs, bundle.jobs.map(sanitizeRuntimeJobForExchange), (job) => job.id, (didChange) => {
+        changed ||= didChange;
+    }, runtimeJobsEqualForExchange).sort((left, right) => (left.startedAt ?? left.createdAt).localeCompare(right.startedAt ?? right.createdAt) ||
+        left.id.localeCompare(right.id));
+    await saveLocalArchive(workspace, {
+        activeId,
+        candidates,
+        candidateFiles,
+        evaluations,
+        runs,
+        events,
+    });
+    await writeArchivedLocalJobs(workspace, jobs, executionFilesByJobId);
+    return {
+        changed,
+        stats: runtimeBundleStats({
+            schema: "workbench.runtime.bundle.v1",
+            activeId,
+            candidates,
+            candidateFiles: Object.entries(candidateFiles).map(([candidateId, files]) => ({
+                candidateId,
+                files,
+            })),
+            evaluations,
+            runs,
+            jobs,
+            executionFiles: [...executionFilesByJobId.entries()].map(([jobId, files]) => ({
+                jobId,
+                files,
+            })),
+            events,
+        }),
+    };
+}
+export function runtimeBundleStats(bundle) {
+    return workbenchRuntimeBundleStats(bundle);
+}
+export function sanitizeRuntimeJobForExchange(job) {
+    return sanitizeWorkbenchRuntimeJobForExchange(job);
+}
+function sanitizeRuntimeJobForArchive(job) {
+    const { leaseUntil: _leaseUntil, wakeupLeaseUntil: _wakeupLeaseUntil, hostId: _hostId, workerId: _workerId, claimTokenHash: _claimTokenHash, ...portable } = job;
+    return { ...portable };
+}
+async function writeArchivedLocalJobs(workspace, jobs, executionFilesByJobId) {
     if (jobs.length === 0) {
         return;
     }
@@ -76,14 +213,18 @@ export async function saveLocalJobs(workspace, jobs) {
         fs.mkdir(executionFilesDir, { recursive: true }),
     ]);
     for (const job of jobs) {
+        const sanitizedJob = sanitizeRuntimeJobForArchive(job);
         const safeJobId = localRecordName(job.id);
-        const traceSourceFiles = filterArchivedExecutionFiles(completedJobOutputFiles(job));
-        const outputFiles = selectExecutionOutputFilesForInspection({
-            purpose: readExecutionPurpose(job),
-            files: traceSourceFiles,
-            output: jsonRecord(job.output),
-        });
-        await writeJson(path.join(jobsDir, `${safeJobId}.json`), archivedLocalJob(job, outputFiles, traceSourceFiles));
+        const explicitOutputFiles = executionFilesByJobId.get(job.id);
+        const traceSourceFiles = filterArchivedExecutionFiles(completedJobOutputFiles(sanitizedJob));
+        const outputFiles = explicitOutputFiles
+            ? copySurfaceFiles(explicitOutputFiles)
+            : selectExecutionOutputFilesForInspection({
+                purpose: readExecutionPurpose(sanitizedJob),
+                files: traceSourceFiles,
+                output: jsonRecord(sanitizedJob.output),
+            });
+        await writeJson(path.join(jobsDir, `${safeJobId}.json`), archivedLocalJob(sanitizedJob, outputFiles, traceSourceFiles.length > 0 ? traceSourceFiles : outputFiles));
         const filesRoot = path.join(executionFilesDir, safeJobId);
         await fs.rm(filesRoot, { force: true, recursive: true });
         await writeSurfaceFiles(filesRoot, outputFiles);
@@ -92,17 +233,17 @@ export async function saveLocalJobs(workspace, jobs) {
 export async function readLocalExecutionFiles(workspace, jobId) {
     return await readSurfaceFiles(path.join(localRuntimeDir(workspace), "execution-files", localRecordName(jobId)));
 }
-export async function readLocalSubjectRecord(workspace, subjectId) {
-    const subject = await readJson(path.join(localRuntimeDir(workspace), "subjects", localRecordName(subjectId), "record.json"), null);
-    if (!subject) {
-        throw new Error(`Subject not found: ${subjectId}`);
+export async function readLocalCandidateRecord(workspace, candidateId) {
+    const candidate = await readJson(path.join(localRuntimeDir(workspace), CANDIDATE_RECORDS_DIR, localRecordName(candidateId), "record.json"), null);
+    if (!candidate) {
+        throw new Error(`Candidate not found: ${candidateId}`);
     }
-    validateSubjectRecord(subject);
-    return subject;
+    validateCandidateRecord(candidate);
+    return candidate;
 }
-export async function readLocalSubjectFilesForId(workspace, subjectId) {
-    await readLocalSubjectRecord(workspace, subjectId);
-    return await readSurfaceFiles(path.join(localRuntimeDir(workspace), "subjects", localRecordName(subjectId), "files"));
+export async function readLocalCandidateFilesForId(workspace, candidateId) {
+    await readLocalCandidateRecord(workspace, candidateId);
+    return await readSurfaceFiles(path.join(localRuntimeDir(workspace), CANDIDATE_RECORDS_DIR, localRecordName(candidateId), "files"));
 }
 export async function readLocalEvaluationRecord(workspace, evaluationId) {
     const evaluation = await readJson(path.join(localRuntimeDir(workspace), "evaluations", `${localRecordName(evaluationId)}.json`), null);
@@ -131,16 +272,16 @@ export async function readLocalRunJobs(workspace, runId) {
 export async function readLocalJobInRun(workspace, runId, jobId) {
     return (await readLocalRunJobs(workspace, runId)).find((job) => job.id === jobId) ?? null;
 }
-export function upsertLocalSubject(snapshot, subject, files) {
+export function upsertLocalCandidate(snapshot, candidate, files) {
     return {
         ...snapshot,
-        subjects: [
-            ...snapshot.subjects.filter((entry) => entry.id !== subject.id),
-            subject,
-        ].sort((left, right) => left.ordinal - right.ordinal || left.id.localeCompare(right.id)),
-        subjectFiles: {
-            ...snapshot.subjectFiles,
-            [subject.id]: files.map((file) => ({ ...file })),
+        candidates: [
+            ...snapshot.candidates.filter((entry) => entry.id !== candidate.id),
+            candidate,
+        ].sort(compareLocalCandidateRecords),
+        candidateFiles: {
+            ...snapshot.candidateFiles,
+            [candidate.id]: files.map((file) => ({ ...file })),
         },
     };
 }
@@ -153,7 +294,7 @@ export function upsertLocalEvaluation(snapshot, evaluation) {
         ].sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id)),
     };
 }
-export function appendLocalRun(snapshot, run, events) {
+export function upsertLocalRun(snapshot, run, events) {
     return {
         ...snapshot,
         runs: [
@@ -172,66 +313,132 @@ export function setLocalActive(snapshot, activeId) {
         activeId,
     };
 }
-export function readLocalSubject(snapshot, subjectId) {
-    const subject = snapshot.subjects.find((entry) => entry.id === subjectId);
-    if (!subject) {
-        throw new Error(`Subject not found: ${subjectId}`);
+export function readLocalCandidate(snapshot, candidateId) {
+    const candidate = snapshot.candidates.find((entry) => entry.id === candidateId);
+    if (!candidate) {
+        throw new Error(`Candidate not found: ${candidateId}`);
     }
-    return subject;
+    return candidate;
 }
-export function readLocalSubjectFiles(snapshot, subjectId) {
-    readLocalSubject(snapshot, subjectId);
-    return (snapshot.subjectFiles[subjectId] ?? []).map((file) => ({ ...file }));
+export function readLocalCandidateFiles(snapshot, candidateId) {
+    readLocalCandidate(snapshot, candidateId);
+    return (snapshot.candidateFiles[candidateId] ?? []).map((file) => ({ ...file }));
 }
 function validateLocalArchiveSnapshot(snapshot) {
     validateLocalArchiveIndex(snapshot);
 }
+function validateRuntimeBundleSchema(bundle) {
+    if (!bundle || bundle.schema !== "workbench.runtime.bundle.v1") {
+        throw new Error("Unsupported Workbench runtime bundle.");
+    }
+}
+function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtimeRecordsEqual) {
+    const records = new Map();
+    for (const record of existing) {
+        records.set(localRecordName(idFor(record)), record);
+    }
+    for (const record of incoming) {
+        const id = localRecordName(idFor(record));
+        const previous = records.get(id);
+        if (!previous || !equal(previous, record)) {
+            if (previous) {
+                throw new Error(`Runtime history conflict for id ${id}.`);
+            }
+            markChanged(true);
+        }
+        records.set(id, record);
+    }
+    return [...records.values()];
+}
+function runtimeRecordsEqual(left, right) {
+    return JSON.stringify(canonicalRuntimeJson(left)) ===
+        JSON.stringify(canonicalRuntimeJson(right));
+}
+function runtimeJobsEqualForExchange(left, right) {
+    return runtimeRecordsEqual(runtimeComparableJob(left), runtimeComparableJob(right));
+}
+function runtimeComparableJob(job) {
+    const comparable = sanitizeRuntimeJobForExchange(job);
+    const output = comparable.output;
+    if (!output || typeof output !== "object" || Array.isArray(output)) {
+        return comparable;
+    }
+    const { files: _files, fileSet: _fileSet, ...portableOutput } = output;
+    return {
+        ...comparable,
+        output: portableOutput,
+    };
+}
+function canonicalRuntimeJson(value) {
+    if (Array.isArray(value)) {
+        return value.map(canonicalRuntimeJson);
+    }
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.keys(value)
+            .sort()
+            .map((key) => [key, canonicalRuntimeJson(value[key])]));
+    }
+    return value;
+}
+function runtimeEventKey(event) {
+    return [
+        event.runId ?? "_",
+        event.jobId ?? "_",
+        event.at,
+        event.id,
+    ].join("#");
+}
+function copySurfaceFiles(files) {
+    return files.map((file) => ({ ...file }));
+}
 function validateLocalArchiveIndex(snapshot) {
-    const subjectIds = new Set(snapshot.subjects.map((subject) => subject.id));
-    if (snapshot.activeId && !subjectIds.has(snapshot.activeId)) {
-        throw new Error(`Active subject not found: ${snapshot.activeId}`);
-    }
-    for (const subject of snapshot.subjects) {
-        validateSubjectRecord(subject);
-        if (!Array.isArray(subject.referenceIds)) {
-            throw new Error(`subject ${subject.id}.referenceIds must be an array.`);
+    const candidateIds = new Set(snapshot.candidates.map((candidate) => candidate.id));
+    if (snapshot.activeId && !candidateIds.has(snapshot.activeId)) {
+        throw new Error(`Active candidate not found: ${snapshot.activeId}`);
+    }
+    for (const candidate of snapshot.candidates) {
+        validateCandidateRecord(candidate);
+        if (!Array.isArray(candidate.referenceIds)) {
+            throw new Error(`candidate ${candidate.id}.referenceIds must be an array.`);
         }
-        if (!Array.isArray(subject.fileChanges)) {
-            throw new Error(`subject ${subject.id}.fileChanges must be an array.`);
+        if (!Array.isArray(candidate.fileChanges)) {
+            throw new Error(`candidate ${candidate.id}.fileChanges must be an array.`);
         }
-        if (subject.baseId && !subjectIds.has(subject.baseId)) {
-            throw new Error(`subject ${subject.id}.baseId not found: ${subject.baseId}`);
+        if (candidate.baseId && !candidateIds.has(candidate.baseId)) {
+            throw new Error(`candidate ${candidate.id}.baseId not found: ${candidate.baseId}`);
         }
     }
     for (const evaluation of snapshot.evaluations) {
         validateEvaluationRecord(evaluation);
-        const subject = snapshot.subjects.find((entry) => entry.id === evaluation.subjectId);
-        if (!subject) {
-            throw new Error(`evaluation ${evaluation.id}.subjectId not found: ${evaluation.subjectId}`);
+        const candidate = snapshot.candidates.find((entry) => entry.id === evaluation.candidateId);
+        if (!candidate) {
+            throw new Error(`evaluation ${evaluation.id}.candidateId not found: ${evaluation.candidateId}`);
         }
-        if (subject.benchmarkFingerprint !== evaluation.benchmarkFingerprint) {
-            throw new Error(`evaluation ${evaluation.id}.benchmarkFingerprint does not match subject ${subject.id}.`);
+        if (candidate.benchmarkFingerprint !== evaluation.benchmarkFingerprint) {
+            throw new Error(`evaluation ${evaluation.id}.benchmarkFingerprint does not match candidate ${candidate.id}.`);
         }
-        if (subject.subjectFingerprint !== evaluation.subjectFingerprint) {
-            throw new Error(`evaluation ${evaluation.id}.subjectFingerprint does not match subject ${subject.id}.`);
+        if (candidate.candidateFingerprint !== evaluation.candidateFingerprint) {
+            throw new Error(`evaluation ${evaluation.id}.candidateFingerprint does not match candidate ${candidate.id}.`);
         }
     }
     for (const run of snapshot.runs) {
         validateRunRecord(run);
     }
 }
-function validateSubjectRecord(subject) {
-    requireArchiveString(subject.id, "subject.id");
-    requireArchiveString(subject.benchmarkFingerprint, `subject ${subject.id}.benchmarkFingerprint`);
-    requireArchiveString(subject.subjectFingerprint, `subject ${subject.id}.subjectFingerprint`);
-    requireArchiveString(subject.createdAt, `subject ${subject.id}.createdAt`);
+function validateCandidateRecord(candidate) {
+    requireArchiveString(candidate.id, "candidate.id");
+    requireArchivePositiveInteger(candidate.version, `candidate ${candidate.id}.version`);
+    requireArchivePositiveInteger(candidate.ordinal, `candidate ${candidate.id}.ordinal`);
+    requireArchiveString(candidate.benchmarkFingerprint, `candidate ${candidate.id}.benchmarkFingerprint`);
+    requireArchiveString(candidate.candidateFingerprint, `candidate ${candidate.id}.candidateFingerprint`);
+    requireArchiveString(candidate.createdAt, `candidate ${candidate.id}.createdAt`);
 }
 function validateEvaluationRecord(evaluation) {
     requireArchiveString(evaluation.id, "evaluation.id");
     requireArchiveString(evaluation.runId, `evaluation ${evaluation.id}.runId`);
     requireArchiveString(evaluation.benchmarkFingerprint, `evaluation ${evaluation.id}.benchmarkFingerprint`);
-    requireArchiveString(evaluation.subjectFingerprint, `evaluation ${evaluation.id}.subjectFingerprint`);
-    requireArchiveString(evaluation.subjectId, `evaluation ${evaluation.id}.subjectId`);
+    requireArchiveString(evaluation.candidateFingerprint, `evaluation ${evaluation.id}.candidateFingerprint`);
+    requireArchiveString(evaluation.candidateId, `evaluation ${evaluation.id}.candidateId`);
 }
 function validateRunRecord(run) {
     requireArchiveString(run.id, "run.id");
@@ -245,18 +452,53 @@ function requireArchiveString(value, label) {
         throw new Error(`${label} must be a non-empty string.`);
     }
 }
+function requireArchivePositiveInteger(value, label) {
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
+        throw new Error(`${label} must be a positive integer.`);
+    }
+}
+function compareLocalCandidateRecords(left, right) {
+    return left.version - right.version ||
+        left.createdAt.localeCompare(right.createdAt) ||
+        left.id.localeCompare(right.id);
+}
 function archivedLocalJob(job, outputFiles, traceSourceFiles) {
     const output = jsonRecord(job.output);
-    const traceSessions = buildLocalJobTraceSessions(job, traceSourceFiles);
+    const existingTrace = readExistingTrace(job);
+    const existingTraceSessions = readExistingTraceSessions(job);
+    const traceSessions = existingTraceSessions.length > 0
+        ? existingTraceSessions
+        : buildLocalJobTraceSessions(job, traceSourceFiles);
     return {
         ...job,
         ...(Object.keys(output).length > 0
-            ? { output: { ...output, files: outputFiles } }
+            ? { output: { ...output, files: traceSourceFiles } }
             : {}),
-        trace: buildLocalJobTrace(job),
+        trace: existingTrace ?? buildLocalJobTrace(job),
         traceSessions,
     };
 }
+function readExistingTrace(job) {
+    const trace = job.trace;
+    if (!trace || typeof trace !== "object" || Array.isArray(trace)) {
+        return null;
+    }
+    return {
+        trace_id: typeof trace.trace_id === "string" && trace.trace_id.length > 0
+            ? trace.trace_id
+            : job.id,
+        spans: Array.isArray(trace.spans) ? trace.spans : [],
+        events: Array.isArray(trace.events) ? trace.events : [],
+        summaries: Array.isArray(trace.summaries) ? trace.summaries : [],
+    };
+}
+function readExistingTraceSessions(job) {
+    const sessions = job.traceSessions;
+    if (!Array.isArray(sessions)) {
+        return [];
+    }
+    return sessions.map((session) => ({ ...session }));
+}
 function filterArchivedExecutionFiles(files) {
     return files.filter((file) => file.path.startsWith(".workbench/traces/") ||
         !isWorkbenchReservedArchivePath(file.path));
@@ -266,7 +508,7 @@ function isWorkbenchReservedArchivePath(filePath) {
 }
 function buildLocalJobTrace(job) {
     const purpose = readExecutionPurpose(job);
-    const role = purpose === "improve" ? "optimizer" : "engine";
+    const role = purpose === "improve" ? "improver" : "engine";
     const stageId = purpose ?? "execution";
     const status = traceStatusForJob(job.status);
     const startedAt = job.startedAt ?? job.createdAt;
@@ -358,7 +600,7 @@ function buildLocalJobTraceSessions(job, outputFiles) {
         job,
         files: outputFiles,
         purpose,
-        fallbackRole: purpose === "improve" ? "optimizer" : "engine",
+        fallbackRole: purpose === "improve" ? "improver" : "engine",
     });
 }
 function completedJobOutputFiles(job) {
@@ -439,8 +681,8 @@ function traceUsageSummary(value) {
     const record = jsonRecord(value);
     const usage = Object.keys(jsonRecord(record.total)).length > 0
         ? jsonRecord(record.total)
-        : Object.keys(jsonRecord(record.optimizer)).length > 0
-            ? jsonRecord(record.optimizer)
+        : Object.keys(jsonRecord(record.improver)).length > 0
+            ? jsonRecord(record.improver)
             : Object.keys(jsonRecord(record.runner)).length > 0
                 ? jsonRecord(record.runner)
                 : Object.keys(jsonRecord(record.engine)).length > 0
@@ -488,8 +730,8 @@ function localRecordName(value) {
     }
     return value;
 }
-export async function materializeSubjectRoot(workspace, subjectRoot, files) {
-    const root = path.join(workspace, normalizeRelativePath(subjectRoot));
+export async function materializeCandidateRoot(workspace, candidateRoot, files) {
+    const root = path.join(workspace, normalizeRelativePath(candidateRoot));
     const before = new Set((await readSurfaceFiles(root)).map((file) => file.path));
     await fs.rm(root, { force: true, recursive: true });
     await writeSurfaceFiles(root, files);

package/dist/project-source.d.ts

@@ -3,9 +3,8 @@ import { type WorkbenchEngineCase, type WorkbenchEngineResolveResult } from "@wo
 import { type WorkspaceSnapshotFile } from "./workspace-snapshot.js";
 import { type ResolvedWorkbenchAdapter } from "./adapter-project.js";
 export declare const WORKBENCH_BENCHMARK_FILE = "benchmark.yaml";
-export declare const WORKBENCH_SUBJECTS_DIR = "subjects";
-export declare const WORKBENCH_OPTIMIZERS_DIR = "optimizers";
-export declare const WORKBENCH_SUBJECT_FILE = "subject.yaml";
+export declare const WORKBENCH_CANDIDATES_DIR = "candidates";
+export declare const WORKBENCH_CANDIDATE_FILE = "candidate.yaml";
 export type HostedFile = WorkspaceSnapshotFile;
 export interface LocalProjectSource {
     dir: string;
@@ -14,20 +13,20 @@ export interface LocalProjectSource {
     spec: ReturnType<typeof resolveWorkbenchResolvedSourceYaml>;
     benchmarkPath: string;
     benchmarkSource: string;
-    subjectName: string;
-    subjectDir: string;
-    subjectFilesPath: string;
-    subjectSpecPath: string;
-    subjectSource: string;
-    optimizerPath?: string;
-    optimizerSource?: string;
+    candidateName: string;
+    candidateDir: string;
+    candidateFilesPath: string;
+    candidateSpecPath: string;
+    candidateSource: string;
+    candidateRunId: string;
+    candidateRunIds: string[];
     benchmarkAdapterSources: string[];
     benchmarkAdapterIds: string[];
     dockerfilePath: string;
     dockerfile: string;
     runtimeDockerfile: string;
     dockerfileFiles: HostedFile[];
-    subjectFiles: HostedFile[];
+    candidateFiles: HostedFile[];
     engineResolveFiles: HostedFile[];
     adapters: ResolvedWorkbenchAdapter[];
     adapterFiles: HostedFile[];
@@ -44,11 +43,9 @@ export interface LocalAuthoredProjectSource {
     specSource: string;
     benchmarkPath: string;
     benchmarkSource: string;
-    subjectDir: string;
-    subjectSpecPath: string;
-    subjectSource: string;
-    optimizerPath?: string;
-    optimizerSource?: string;
+    candidateDir: string;
+    candidateSpecPath: string;
+    candidateSource: string;
     sourceFiles: SurfaceSnapshotFile[];
 }
 export interface LocalEngineResolveInvocation {
@@ -57,7 +54,7 @@ export interface LocalEngineResolveInvocation {
     auth?: Json;
 }
 interface LocalProjectSourceOptions {
-    optimizerPath?: string;
+    runId?: string;
 }
 export declare function readLocalProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalProjectSource>;
 export declare function readLocalAuthoredProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalAuthoredProjectSource>;

package/dist/project-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAML,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAQL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,sBAAsB,aAAa,CAAC;AACjD,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,sBAAsB,iBAAiB,CAAC;AAErD,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CAsH7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CAqCrC"}
+{"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAQL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CA8G7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CA6BrC"}