@workbench-ai/workbench-core 0.0.66 → 0.0.68

package/dist/generic-spec.js

@@ -1,8 +1,8 @@
 import { createHash } from "node:crypto";
 import { isWorkbenchExecutionNetworkEgress, } from "@workbench-ai/workbench-contract";
 import YAML from "yaml";
-export const BENCHMARK_SPEC_FILE = "benchmark.yaml";
-export const CANDIDATE_SPEC_FILE = "candidate.yaml";
+export const EVAL_SPEC_FILE = "eval.yaml";
+export const SKILL_SPEC_FILE = "skill.yaml";
 export const DEFAULT_EXECUTION_RESOURCES = {
     cpu: 2,
     memoryGb: 4,
@@ -14,7 +14,7 @@ export function validateWorkbenchResolvedSourceYaml(source) {
     const warnings = [];
     const trimmed = source.trim();
     if (!trimmed) {
-        errors.push("Resolved Workbench source cannot be empty.");
+        errors.push("Resolved Workbench spec cannot be empty.");
     }
     if (trimmed) {
         try {
@@ -31,25 +31,25 @@ export function validateWorkbenchResolvedSourceYaml(source) {
     };
 }
 export function resolveWorkbenchResolvedSourceYaml(source) {
-    const parsed = parseYamlRecord(source, "resolved Workbench source");
+    const parsed = parseYamlRecord(source, "resolved Workbench spec");
     const errors = [];
-    rejectUnknownKeys(parsed, "resolved Workbench source", [
+    rejectUnknownKeys(parsed, "resolved Workbench spec", [
         "version",
-        "benchmark",
-        "candidate",
+        "eval",
+        "skill",
     ], errors);
     if (parsed.version !== 4) {
-        throw new Error("Resolved Workbench source version must be 4.");
+        throw new Error("Resolved Workbench spec version must be 4.");
     }
-    const benchmark = normalizeBenchmarkRecord(readRequiredRecord(parsed.benchmark, "resolved Workbench source.benchmark", errors), "benchmark.yaml", errors);
-    const candidate = normalizeCandidateRecord(readRequiredRecord(parsed.candidate, "resolved Workbench source.candidate", errors), "resolved Workbench source.candidate", errors);
+    const evalSpec = normalizeEvalRecord(readRequiredRecord(parsed.eval, EVAL_SPEC_FILE, errors), EVAL_SPEC_FILE, "resolved", errors);
+    const skill = normalizeSkillRecord(readRequiredRecord(parsed.skill, "resolved Workbench spec.skill", errors), "resolved Workbench spec.skill", "resolved", errors);
     if (errors.length > 0) {
         throw new Error(errors.join("\n"));
     }
     return genericSpecFromAuthoredBundle({
         version: 4,
-        benchmark: benchmark,
-        candidate: candidate,
+        eval: evalSpec,
+        skill: skill,
     });
 }
 export function engineResolveBindingForSourceYaml(source) {
@@ -58,7 +58,7 @@ export function engineResolveBindingForSourceYaml(source) {
 export function engineResolveBindingForSpec(spec) {
     const resolver = engineResolveInvocationForSpec(spec);
     return {
-        engine: spec.benchmark.engine.use,
+        engine: spec.eval.engine.use,
         resolver: {
             use: resolver.use,
             withFingerprint: fingerprintJson(resolver.with ?? {}),
@@ -67,29 +67,29 @@ export function engineResolveBindingForSpec(spec) {
 }
 export function resolveWorkbenchSourceFiles(args) {
     return genericSpecFromAuthoredBundle(parseWorkbenchSourceFiles({
-        benchmarkSource: args.benchmarkSource,
-        candidateSource: args.candidateSource,
-        runId: args.runId,
+        evalSource: args.evalSource,
+        skillSource: args.skillSource,
+        selectedAgentId: args.selectedAgentId,
     }));
 }
 export function parseWorkbenchSourceFiles(args) {
     const errors = [];
-    const benchmark = normalizeBenchmarkRecord(parseYamlRecord(args.benchmarkSource, BENCHMARK_SPEC_FILE), BENCHMARK_SPEC_FILE, errors);
-    const candidate = normalizeCandidateRecord(parseYamlRecord(args.candidateSource ?? "", "candidate YAML"), "candidate YAML", errors, args.runId ?? undefined);
+    const evalSpec = normalizeEvalRecord(parseYamlRecord(args.evalSource, EVAL_SPEC_FILE), EVAL_SPEC_FILE, "authored", errors);
+    const skill = normalizeSkillRecord(parseYamlRecord(args.skillSource ?? "", "skill YAML"), "skill YAML", "authored", errors, args.selectedAgentId ?? undefined);
     if (errors.length > 0) {
         throw new Error(errors.join("\n"));
     }
     return {
         version: 4,
-        benchmark: benchmark,
-        candidate: candidate,
+        eval: evalSpec,
+        skill: skill,
     };
 }
 export function serializeWorkbenchResolvedSourceYaml(source) {
     return YAML.stringify(source).trimEnd() + "\n";
 }
-export function isWorkbenchCandidateManifestPath(filePath) {
-    return /^candidates\/[^/]+\/candidate\.ya?ml$/iu.test(filePath.replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/^(?:\.\/)+/u, ""));
+export function isWorkbenchSkillManifestPath(filePath) {
+    return /^skills\/[^/]+\/skill\.ya?ml$/iu.test(filePath.replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/^(?:\.\/)+/u, ""));
 }
 export function resolveEngineCaseExecutionConfig(args) {
     return {
@@ -131,38 +131,38 @@ export function runtimeSandboxRef(runtime) {
     return `dockerfile://${runtime.dockerfile}`;
 }
 function genericSpecFromAuthoredBundle(source) {
-    const engineRuntime = engineRuntimeFromConfig(source.benchmark.engine);
-    const engineRun = cloneEngineInvocation(source.benchmark.engine);
-    const engineResolve = cloneEngineInvocation(source.benchmark.engine);
-    const candidate = source.candidate;
-    const selectedRun = candidate.runs[candidate.selectedRunId];
-    if (!selectedRun) {
-        throw new Error(`Candidate run not found: ${candidate.selectedRunId}`);
+    const engineRuntime = engineRuntimeFromConfig(source.eval.engine);
+    const engineRun = cloneEngineInvocation(source.eval.engine);
+    const engineResolve = cloneEngineInvocation(source.eval.engine);
+    const skill = source.skill;
+    const selectedAgent = skill.agents[skill.selectedAgentId];
+    if (!selectedAgent) {
+        throw new Error(`Skill agent not found: ${skill.selectedAgentId}`);
     }
     return {
         version: 4,
-        name: source.benchmark.name,
-        description: source.benchmark.description,
-        benchmark: {
-            name: source.benchmark.name,
-            description: source.benchmark.description,
-            engine: cloneJson(source.benchmark.engine),
+        name: source.eval.name,
+        description: source.eval.description,
+        eval: {
+            name: source.eval.name,
+            description: source.eval.description,
+            engine: cloneJson(source.eval.engine),
         },
-        candidate: {
-            name: candidate.name,
-            ...(candidate.description ? { description: candidate.description } : {}),
-            files: cloneJson(candidate.files),
-            ...(candidate.prepare ? { prepare: cloneJson(candidate.prepare) } : {}),
-            defaultRun: candidate.defaultRun ?? candidate.selectedRunId,
-            selectedRunId: candidate.selectedRunId,
-            selectedRunName: selectedRun.name,
-            runs: cloneJson(candidate.runs),
-            ...(candidate.improve
+        skill: {
+            name: skill.name,
+            ...(skill.description ? { description: skill.description } : {}),
+            files: cloneJson(skill.files),
+            ...(skill.prepare ? { prepare: cloneJson(skill.prepare) } : {}),
+            defaultAgent: skill.defaultAgent ?? skill.selectedAgentId,
+            selectedAgentId: skill.selectedAgentId,
+            selectedAgentName: selectedAgent.name,
+            agents: cloneJson(skill.agents),
+            ...(skill.improve
                 ? {
                     improve: {
-                        edits: [...candidate.improve.edits],
-                        ...(candidate.improve.optimizeOn ? { optimizeOn: cloneJson(candidate.improve.optimizeOn) } : {}),
-                        ...(candidate.improve.selectBy ? { selectBy: cloneJson(candidate.improve.selectBy) } : {}),
+                        edits: [...skill.improve.edits],
+                        ...(skill.improve.optimizeOn ? { optimizeOn: cloneJson(skill.improve.optimizeOn) } : {}),
+                        ...(skill.improve.selectBy ? { selectBy: cloneJson(skill.improve.selectBy) } : {}),
                     },
                 }
                 : {}),
@@ -170,18 +170,18 @@ function genericSpecFromAuthoredBundle(source) {
         environment: cloneJson(engineRuntime),
         adapters: [
             ...new Set([
-                ...source.benchmark.adapters,
-                ...candidate.adapters,
+                ...source.eval.adapters,
+                ...skill.adapters,
             ]),
         ],
-        engine: cloneJson(source.benchmark.engine),
+        engine: cloneJson(source.eval.engine),
         engineResolve: cloneJson(engineResolve),
-        ...(candidate.improve ? { improve: clonePhaseAdapter(candidate.improve) } : {}),
-        run: clonePhaseAdapter(selectedRun),
+        ...(skill.improve ? { improve: clonePhaseAdapter(skill.improve) } : {}),
+        run: clonePhaseAdapter(selectedAgent),
         engineRun: cloneJson(engineRun),
     };
 }
-function normalizeBenchmarkRecord(record, label, errors) {
+function normalizeEvalRecord(record, label, mode, errors) {
     if (!record) {
         return null;
     }
@@ -192,7 +192,7 @@ function normalizeBenchmarkRecord(record, label, errors) {
         "adapters",
         "engine",
     ], errors);
-    requireVersionFour(record.version, label, errors);
+    requireSpecVersion(record.version, label, mode === "authored" ? 1 : 4, errors);
     const name = readRequiredString(record.name, `${label}.name`, errors);
     const description = readRequiredString(record.description, `${label}.description`, errors);
     const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
@@ -222,7 +222,7 @@ function normalizeEngineRuntimeConfig(engine, label, errors) {
         }
     }
 }
-function normalizeCandidateRecord(record, label, errors, selectedRunId) {
+function normalizeSkillRecord(record, label, mode, errors, selectedAgentId) {
     if (!record) {
         return null;
     }
@@ -233,26 +233,28 @@ function normalizeCandidateRecord(record, label, errors, selectedRunId) {
         "files",
         "prepare",
         "adapters",
-        "defaultRun",
-        "runs",
+        "defaultAgent",
+        "agents",
+        ...(mode === "resolved" ? ["selectedAgentId"] : []),
         "improve",
-        "selectedRunId",
     ], errors);
-    requireVersionFour(record.version, label, errors);
+    requireSpecVersion(record.version, label, mode === "authored" ? 1 : 4, errors);
     const name = readRequiredString(record.name, `${label}.name`, errors);
     const description = readOptionalString(record.description, `${label}.description`, errors);
     const files = normalizePathRef(record.files, `${label}.files`, errors);
-    const prepare = normalizeCandidatePrepare(record.prepare, `${label}.prepare`, errors);
+    const prepare = normalizeSkillPrepare(record.prepare, `${label}.prepare`, errors);
     const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
-    const runs = normalizeCandidateRuns(record.runs, `${label}.runs`, errors);
-    const defaultRun = readOptionalString(record.defaultRun, `${label}.defaultRun`, errors);
-    const embeddedSelectedRun = readOptionalString(record.selectedRunId, `${label}.selectedRunId`, errors);
-    const selected = selectedRunId ?? embeddedSelectedRun ?? defaultRun ?? Object.keys(runs).sort()[0];
-    if (selected && !runs[selected]) {
-        errors.push(`${label}.selectedRunId references unknown run ${selected}.`);
-    }
-    const improve = normalizeCandidateImprove(record.improve, `${label}.improve`, errors);
-    return name && files && selected && Object.keys(runs).length > 0
+    const agents = normalizeSkillAgents(record.agents, `${label}.agents`, errors);
+    const defaultAgent = readOptionalString(record.defaultAgent, `${label}.defaultAgent`, errors);
+    const embeddedSelectedAgent = mode === "resolved"
+        ? readOptionalString(record.selectedAgentId, `${label}.selectedAgentId`, errors)
+        : undefined;
+    const selected = selectedAgentId ?? embeddedSelectedAgent ?? defaultAgent ?? Object.keys(agents).sort()[0];
+    if (selected && !agents[selected]) {
+        errors.push(`${label}.${mode === "authored" ? "defaultAgent" : "selectedAgentId"} references unknown agent ${selected}.`);
+    }
+    const improve = normalizeSkillImprove(record.improve, `${label}.improve`, errors);
+    return name && files && selected && Object.keys(agents).length > 0
         ? {
             version: 4,
             name,
@@ -260,14 +262,14 @@ function normalizeCandidateRecord(record, label, errors, selectedRunId) {
             files,
             ...(prepare ? { prepare } : {}),
             adapters,
-            ...(defaultRun ? { defaultRun } : {}),
-            runs,
+            ...(defaultAgent ? { defaultAgent } : {}),
+            agents,
             ...(improve ? { improve } : {}),
-            selectedRunId: selected,
+            selectedAgentId: selected,
         }
         : null;
 }
-function normalizeCandidatePrepare(value, label, errors) {
+function normalizeSkillPrepare(value, label, errors) {
     if (value === undefined) {
         return undefined;
     }
@@ -279,37 +281,37 @@ function normalizeCandidatePrepare(value, label, errors) {
     const command = readRequiredString(record.command, `${label}.command`, errors);
     return command ? { command } : undefined;
 }
-function normalizeCandidateRuns(value, label, errors) {
+function normalizeSkillAgents(value, label, errors) {
     const record = readRequiredRecord(value, label, errors);
     if (!record) {
         return {};
     }
-    const runs = {};
-    for (const [runId, runValue] of Object.entries(record).sort(([left], [right]) => left.localeCompare(right))) {
-        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/u.test(runId)) {
-            errors.push(`${label}.${runId} must use letters, numbers, dots, underscores, or dashes.`);
+    const agents = {};
+    for (const [agentId, agentValue] of Object.entries(record).sort(([left], [right]) => left.localeCompare(right))) {
+        if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/u.test(agentId)) {
+            errors.push(`${label}.${agentId} must use letters, numbers, dots, underscores, or dashes.`);
             continue;
         }
-        const runRecord = readRequiredRecord(runValue, `${label}.${runId}`, errors);
-        if (!runRecord) {
+        const agentRecord = readRequiredRecord(agentValue, `${label}.${agentId}`, errors);
+        if (!agentRecord) {
             continue;
         }
-        rejectUnknownKeys(runRecord, `${label}.${runId}`, ["name", "use", "with", "auth"], errors);
-        const name = readRequiredString(runRecord.name, `${label}.${runId}.name`, errors);
-        const invocation = normalizePhaseAdapter(adapterRecordFrom(runRecord), `${label}.${runId}`, errors);
+        rejectUnknownKeys(agentRecord, `${label}.${agentId}`, ["name", "use", "with", "auth"], errors);
+        const name = readRequiredString(agentRecord.name, `${label}.${agentId}.name`, errors);
+        const invocation = normalizePhaseAdapter(adapterRecordFrom(agentRecord), `${label}.${agentId}`, errors);
         if (name && invocation) {
-            runs[runId] = {
+            agents[agentId] = {
                 name,
                 ...invocation,
             };
         }
     }
-    if (Object.keys(runs).length === 0) {
-        errors.push(`${label} must declare at least one run.`);
+    if (Object.keys(agents).length === 0) {
+        errors.push(`${label} must declare at least one agent.`);
     }
-    return runs;
+    return agents;
 }
-function normalizeCandidateImprove(value, label, errors) {
+function normalizeSkillImprove(value, label, errors) {
     if (value === undefined) {
         return undefined;
     }
@@ -385,9 +387,9 @@ function adapterRecordFrom(record) {
         ...(record.auth !== undefined ? { auth: record.auth } : {}),
     };
 }
-function requireVersionFour(value, label, errors) {
-    if (value !== 4) {
-        errors.push(`${label}.version must be 4.`);
+function requireSpecVersion(value, label, version, errors) {
+    if (value !== version) {
+        errors.push(`${label}.version must be ${version}.`);
     }
 }
 function normalizeRuntime(value, label, errors) {