@interf/compiler 0.3.4 → 0.4.0

package/dist/lib/{benchmark-execution.js → test-execution.js}

@@ -2,19 +2,19 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import { buildRuntimeExecutorInfo } from "./executors.js";
-import { benchmarkRunGitignorePath, benchmarkRunsPath, benchmarkSandboxGitignorePath, benchmarkSandboxesPath, normalizeBenchmarkId, } from "./benchmark-paths.js";
-import { createBenchmarkSandbox, } from "./benchmark-sandbox.js";
+import { targetTestRunGitignorePath, targetTestRunsPath, targetTestSandboxGitignorePath, targetTestSandboxesPath, normalizeTestId, } from "./test-paths.js";
+import { createTestSandbox, } from "./test-sandbox.js";
 function parseWords(content) {
     return content.trim().split(/\s+/).filter(Boolean).length;
 }
 function normalizeText(content) {
     return content.toLowerCase().replace(/\s+/g, " ").trim();
 }
-function evaluateTextExpect(benchmarkCase, content) {
+function evaluateTextExpect(testCase, content) {
     const checks = [];
     const normalized = normalizeText(content);
     const wordCount = parseWords(content);
-    for (const phrase of benchmarkCase.expect?.must_include ?? []) {
+    for (const phrase of testCase.expect?.must_include ?? []) {
         const ok = normalized.includes(normalizeText(phrase));
         checks.push({
             label: `must include "${phrase}"`,
@@ -22,7 +22,7 @@ function evaluateTextExpect(benchmarkCase, content) {
             detail: ok ? "present" : "missing",
         });
     }
-    for (const options of benchmarkCase.expect?.must_include_one_of ?? []) {
+    for (const options of testCase.expect?.must_include_one_of ?? []) {
         const ok = options.some((phrase) => normalized.includes(normalizeText(phrase)));
         checks.push({
             label: `must include one of [${options.map((phrase) => `"${phrase}"`).join(", ")}]`,
@@ -30,7 +30,7 @@ function evaluateTextExpect(benchmarkCase, content) {
             detail: ok ? "present" : "missing",
         });
     }
-    for (const phrase of benchmarkCase.expect?.must_not_include ?? []) {
+    for (const phrase of testCase.expect?.must_not_include ?? []) {
         const ok = !normalized.includes(normalizeText(phrase));
         checks.push({
             label: `must not include "${phrase}"`,
@@ -38,18 +38,18 @@ function evaluateTextExpect(benchmarkCase, content) {
             detail: ok ? "absent" : "present",
         });
     }
-    if (typeof benchmarkCase.expect?.min_words === "number") {
-        const ok = wordCount >= benchmarkCase.expect.min_words;
+    if (typeof testCase.expect?.min_words === "number") {
+        const ok = wordCount >= testCase.expect.min_words;
         checks.push({
-            label: `min words ${benchmarkCase.expect.min_words}`,
+            label: `min words ${testCase.expect.min_words}`,
             ok,
             detail: `${wordCount} words`,
         });
     }
-    if (typeof benchmarkCase.expect?.max_words === "number") {
-        const ok = wordCount <= benchmarkCase.expect.max_words;
+    if (typeof testCase.expect?.max_words === "number") {
+        const ok = wordCount <= testCase.expect.max_words;
         checks.push({
-            label: `max words ${benchmarkCase.expect.max_words}`,
+            label: `max words ${testCase.expect.max_words}`,
             ok,
             detail: `${wordCount} words`,
         });
@@ -59,25 +59,25 @@ function evaluateTextExpect(benchmarkCase, content) {
         wordCount,
     };
 }
-function benchmarkCaseNeedsExecutor(benchmarkCase) {
-    return !benchmarkCase.file || Boolean(benchmarkCase.answer);
+function testCaseNeedsExecutor(testCase) {
+    return !testCase.file || Boolean(testCase.answer);
 }
-function runBenchmarkCase(target, benchmarkCase) {
-    if (!benchmarkCase.file) {
-        throw new Error(`Benchmark case "${benchmarkCase.id}" requires an executor because it has no file target.`);
+function runTestCase(target, testCase) {
+    if (!testCase.file) {
+        throw new Error(`Test case "${testCase.id}" requires an executor because it has no file target.`);
     }
-    const outputPath = join(target.path, benchmarkCase.file);
+    const outputPath = join(target.path, testCase.file);
     const checks = [];
     if (!existsSync(outputPath)) {
         checks.push({
             label: "file exists",
             ok: false,
-            detail: `Missing ${benchmarkCase.file}`,
+            detail: `Missing ${testCase.file}`,
         });
         return {
-            caseId: benchmarkCase.id,
-            question: benchmarkCase.question,
-            ...(benchmarkCase.file ? { file: benchmarkCase.file } : {}),
+            caseId: testCase.id,
+            question: testCase.question,
+            ...(testCase.file ? { file: testCase.file } : {}),
             ok: false,
             wordCount: 0,
             passedChecks: 0,
@@ -88,17 +88,17 @@ function runBenchmarkCase(target, benchmarkCase) {
     checks.push({
         label: "file exists",
         ok: true,
-        detail: `Found ${benchmarkCase.file}`,
+        detail: `Found ${testCase.file}`,
     });
     const content = readFileSync(outputPath, "utf8");
-    const evaluated = evaluateTextExpect(benchmarkCase, content);
+    const evaluated = evaluateTextExpect(testCase, content);
     const wordCount = evaluated.wordCount;
     checks.push(...evaluated.checks);
     const passedChecks = checks.filter((check) => check.ok).length;
     return {
-        caseId: benchmarkCase.id,
-        question: benchmarkCase.question,
-        ...(benchmarkCase.file ? { file: benchmarkCase.file } : {}),
+        caseId: testCase.id,
+        question: testCase.question,
+        ...(testCase.file ? { file: testCase.file } : {}),
         ok: passedChecks === checks.length,
         wordCount,
         passedChecks,
@@ -106,25 +106,25 @@ function runBenchmarkCase(target, benchmarkCase) {
         checks,
     };
 }
-function buildBenchmarkJudgePrompt(benchmarkCase, candidateLabel, candidateContent, verdictPath) {
+function buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath) {
     return [
-        "You are judging whether one Interf benchmark answer passes.",
+        "You are judging whether one Interf test answer passes.",
         "Do not browse other files or ask follow-up questions.",
-        "Judge only from the benchmark rule and the candidate answer below.",
+        "Judge only from the truth-check rule and the candidate answer below.",
         "Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
         `Write JSON to ${JSON.stringify(verdictPath)} with keys: pass (boolean), summary (string).`,
         "Before finishing, write the JSON verdict file.",
         "Final line must be `DONE: pass=true - <short summary>` or `DONE: pass=false - <short summary>`.",
-        `Question: ${benchmarkCase.question}`,
-        `Expected answer: ${benchmarkCase.answer ?? "The answer clearly satisfies the question."}`,
-        `Strictness: ${benchmarkCase.strictness ?? "approximate"}`,
+        `Question: ${testCase.question}`,
+        `Expected answer: ${testCase.answer ?? "The answer clearly satisfies the question."}`,
+        `Strictness: ${testCase.strictness ?? "approximate"}`,
         `Candidate: ${candidateLabel}`,
         "Candidate answer starts after the next line and ends at `END CANDIDATE`.",
         candidateContent,
         "END CANDIDATE",
     ].join("\n");
 }
-function readBenchmarkJudgeVerdict(verdictPath) {
+function readTestJudgeVerdict(verdictPath) {
     if (!existsSync(verdictPath))
         return null;
     const raw = JSON.parse(readFileSync(verdictPath, "utf8"));
@@ -133,7 +133,7 @@ function readBenchmarkJudgeVerdict(verdictPath) {
         summary: typeof raw.summary === "string" ? raw.summary : "",
     };
 }
-function readBenchmarkJudgeVerdictFromStatus(statusPath) {
+function readTestJudgeVerdictFromStatus(statusPath) {
     if (!existsSync(statusPath))
         return null;
     const lines = readFileSync(statusPath, "utf8")
@@ -163,14 +163,14 @@ function readBenchmarkJudgeVerdictFromStatus(statusPath) {
     }
     return null;
 }
-async function runBenchmarkJudge(benchmarkCase, executor, candidateLabel, candidateContent) {
-    const tempDir = mkdtempSync(join(tmpdir(), "interf-benchmark-judge-"));
+async function runTargetTestsJudge(testCase, executor, candidateLabel, candidateContent) {
+    const tempDir = mkdtempSync(join(tmpdir(), "interf-test-judge-"));
     let executionError = null;
     let verdict = null;
     try {
         const verdictPath = join(tempDir, "verdict.json");
         const statusPath = join(tempDir, "judge.status.log");
-        const prompt = buildBenchmarkJudgePrompt(benchmarkCase, candidateLabel, candidateContent, verdictPath);
+        const prompt = buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath);
         try {
             await executor.execute(tempDir, prompt, {
                 statusLogPath: statusPath,
@@ -180,9 +180,9 @@ async function runBenchmarkJudge(benchmarkCase, executor, candidateLabel, candid
             executionError = error instanceof Error ? error.message : String(error);
         }
         try {
-            verdict = readBenchmarkJudgeVerdict(verdictPath);
+            verdict = readTestJudgeVerdict(verdictPath);
             if (!verdict) {
-                verdict = readBenchmarkJudgeVerdictFromStatus(statusPath);
+                verdict = readTestJudgeVerdictFromStatus(statusPath);
             }
         }
         catch (error) {
@@ -194,22 +194,23 @@ async function runBenchmarkJudge(benchmarkCase, executor, candidateLabel, candid
     }
     return { verdict, error: executionError };
 }
-function buildBenchmarkQueryPrompt(target, benchmarkCase, answerPath, tracePath) {
+function buildTestQueryPrompt(target, testCase, answerPath, tracePath) {
     const header = target.type === "workspace"
         ? [
-            "You are running an Interf benchmark inside an isolated sandboxed compiled workspace.",
+            "You are running an Interf test inside an isolated sandboxed compiled workspace.",
             "Read `AGENTS.md` first.",
             "Use the local native `interf-query` skill available in this workspace.",
-            "If you need the editable method source, read `workflow/use/query/SKILL.md`.",
-            "Answer the benchmark question the same way you would answer a real user inside this compiled workspace.",
+            "Answer the truth-check question the same way you would answer a real user inside this compiled workspace.",
             "Prefer `home.md`, `knowledge/`, and `summaries/` before raw fallback.",
-            "This sandbox is self-contained: the copied workspace has its own sanitized `raw/` fallback via `interf.json` `source.path`.",
+            "This sandbox is self-contained: the copied workspace has its own sanitized `raw/` fallback via `.interf/interf.json` `source.path`.",
             "The source-folder control plane is intentionally absent from this sandbox. Work only from this sandboxed workspace and its embedded raw files.",
         ]
         : [
-            "You are running an Interf baseline test inside an isolated sandbox of the raw files for this folder.",
+            "You are running an Interf baseline test inside an isolated raw test shell.",
+            "Read `AGENTS.md` first.",
+            "Use the local native `interf-query` skill available in this shell.",
             "There is no compiled workspace in this sandbox.",
-            "Answer only from the sandboxed raw files you can read here.",
+            "Answer only from `raw/` inside this shell.",
             "The source-folder control plane is intentionally absent from this sandbox.",
         ];
     return [
@@ -218,18 +219,18 @@ function buildBenchmarkQueryPrompt(target, benchmarkCase, answerPath, tracePath)
         "Do not ask follow-up questions.",
         `Write the answer to ${JSON.stringify(answerPath)}.`,
         `Write the trace to ${JSON.stringify(tracePath)} with keys: case_id, target, artifacts_consulted, raw_paths_read, used_raw_fallback, answer_summary.`,
-        `Set \`case_id\` to ${JSON.stringify(benchmarkCase.id)}.`,
+        `Set \`case_id\` to ${JSON.stringify(testCase.id)}.`,
         `Set \`target\` to ${JSON.stringify(target.type)}.`,
-        `Question: ${benchmarkCase.question}`,
+        `Question: ${testCase.question}`,
     ].join("\n");
 }
-async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
-    const tempDir = mkdtempSync(join(tmpdir(), "interf-benchmark-live-"));
+async function runLiveTestCase(target, testCase, executor) {
+    const tempDir = mkdtempSync(join(tmpdir(), "interf-test-live-"));
     const answerPath = join(tempDir, "answer.md");
     const tracePath = join(tempDir, "trace.json");
     const statusPath = join(tempDir, "status.log");
     const eventPath = join(tempDir, "events.ndjson");
-    const prompt = buildBenchmarkQueryPrompt(target, benchmarkCase, answerPath, tracePath);
+    const prompt = buildTestQueryPrompt(target, testCase, answerPath, tracePath);
     let executionError = null;
     let code = -1;
     try {
@@ -250,8 +251,8 @@ async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
                 detail: executionError ? `missing answer file (${executionError})` : "missing answer file",
             });
             return {
-                caseId: benchmarkCase.id,
-                question: benchmarkCase.question,
+                caseId: testCase.id,
+                question: testCase.question,
                 ok: false,
                 wordCount: 0,
                 passedChecks: 0,
@@ -260,7 +261,7 @@ async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
             };
         }
         const answer = readFileSync(answerPath, "utf8");
-        const evaluated = evaluateTextExpect(benchmarkCase, answer);
+        const evaluated = evaluateTextExpect(testCase, answer);
         checks.push({
             label: "answer exists",
             ok: true,
@@ -292,8 +293,8 @@ async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
                 detail: "missing trace file",
             });
         }
-        if (benchmarkCase.answer) {
-            const judged = await runBenchmarkJudge(benchmarkCase, executor, `generated answer for ${benchmarkCase.id}`, answer);
+        if (testCase.answer) {
+            const judged = await runTargetTestsJudge(testCase, executor, `generated answer for ${testCase.id}`, answer);
             checks.push({
                 label: "judge verdict",
                 ok: judged.verdict?.pass === true,
@@ -304,8 +305,8 @@ async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
         }
         const passedChecks = checks.filter((check) => check.ok).length;
         return {
-            caseId: benchmarkCase.id,
-            question: benchmarkCase.question,
+            caseId: testCase.id,
+            question: testCase.question,
             ok: code === 0 && passedChecks === checks.length,
             wordCount: evaluated.wordCount,
             passedChecks,
@@ -319,22 +320,22 @@ async function runLiveBenchmarkCase(target, benchmarkCase, executor) {
         rmSync(tempDir, { recursive: true, force: true });
     }
 }
-async function runBenchmarkCaseWithJudge(target, benchmarkCase, executor) {
-    if (!benchmarkCase.file) {
-        return runLiveBenchmarkCase(target, benchmarkCase, executor);
+async function runTestCaseWithJudge(target, testCase, executor) {
+    if (!testCase.file) {
+        return runLiveTestCase(target, testCase, executor);
     }
-    const outputPath = join(target.path, benchmarkCase.file);
+    const outputPath = join(target.path, testCase.file);
     const checks = [];
     if (!existsSync(outputPath)) {
         checks.push({
             label: "file exists",
             ok: false,
-            detail: `Missing ${benchmarkCase.file}`,
+            detail: `Missing ${testCase.file}`,
         });
         return {
-            caseId: benchmarkCase.id,
-            question: benchmarkCase.question,
-            ...(benchmarkCase.file ? { file: benchmarkCase.file } : {}),
+            caseId: testCase.id,
+            question: testCase.question,
+            ...(testCase.file ? { file: testCase.file } : {}),
             ok: false,
             wordCount: 0,
             passedChecks: 0,
@@ -343,16 +344,16 @@ async function runBenchmarkCaseWithJudge(target, benchmarkCase, executor) {
         };
     }
     const content = readFileSync(outputPath, "utf8");
-    const evaluated = evaluateTextExpect(benchmarkCase, content);
+    const evaluated = evaluateTextExpect(testCase, content);
     const wordCount = evaluated.wordCount;
     checks.push({
         label: "file exists",
         ok: true,
-        detail: `Found ${benchmarkCase.file}`,
+        detail: `Found ${testCase.file}`,
     });
     checks.push(...evaluated.checks);
-    if (benchmarkCase.answer) {
-        const judged = await runBenchmarkJudge(benchmarkCase, executor, `compiled file ${outputPath}`, content);
+    if (testCase.answer) {
+        const judged = await runTargetTestsJudge(testCase, executor, `compiled file ${outputPath}`, content);
         checks.push({
             label: "judge verdict",
             ok: judged.verdict?.pass === true,
@@ -363,9 +364,9 @@ async function runBenchmarkCaseWithJudge(target, benchmarkCase, executor) {
     }
     const passedChecks = checks.filter((check) => check.ok).length;
     return {
-        caseId: benchmarkCase.id,
-        question: benchmarkCase.question,
-        ...(benchmarkCase.file ? { file: benchmarkCase.file } : {}),
+        caseId: testCase.id,
+        question: testCase.question,
+        ...(testCase.file ? { file: testCase.file } : {}),
         ok: passedChecks === checks.length,
         wordCount,
         passedChecks,
@@ -373,7 +374,7 @@ async function runBenchmarkCaseWithJudge(target, benchmarkCase, executor) {
         checks,
     };
 }
-function buildBenchmarkTargetResult(target, caseResults, options = {}) {
+function buildTestTargetResult(target, caseResults, options = {}) {
     const passedCases = caseResults.filter((result) => result.ok).length;
     const passedChecks = caseResults.reduce((total, result) => total + result.passedChecks, 0);
     const totalChecks = caseResults.reduce((total, result) => total + result.totalChecks, 0);
@@ -393,25 +394,25 @@ function buildBenchmarkTargetResult(target, caseResults, options = {}) {
         caseResults,
     };
 }
-function buildBenchmarkRunResult(sourcePath, spec, results, executor, generatedAt) {
+function buildTestTargetRun(sourcePath, spec, results, executor, generatedAt) {
     return {
-        kind: "interf-benchmark-run",
+        kind: "interf-test-target-run",
         version: 1,
         generated_at: generatedAt ?? new Date().toISOString(),
-        benchmark: {
+        spec: {
             id: spec.id,
             name: spec.name,
             type: spec.type,
             file: spec.filePath,
             ...(spec.description ? { description: spec.description } : {}),
             case_count: spec.cases.length,
-            cases: spec.cases.map((benchmarkCase) => ({
-                id: benchmarkCase.id,
-                question: benchmarkCase.question,
-                ...(benchmarkCase.file ? { file: benchmarkCase.file } : {}),
-                ...(benchmarkCase.answer ? { answer: benchmarkCase.answer } : {}),
-                ...(benchmarkCase.strictness ? { strictness: benchmarkCase.strictness } : {}),
-                ...(benchmarkCase.expect ? { expect: benchmarkCase.expect } : {}),
+            cases: spec.cases.map((testCase) => ({
+                id: testCase.id,
+                question: testCase.question,
+                ...(testCase.file ? { file: testCase.file } : {}),
+                ...(testCase.answer ? { answer: testCase.answer } : {}),
+                ...(testCase.strictness ? { strictness: testCase.strictness } : {}),
+                ...(testCase.expect ? { expect: testCase.expect } : {}),
             })),
         },
         source_path: sourcePath,
@@ -420,97 +421,99 @@ function buildBenchmarkRunResult(sourcePath, spec, results, executor, generatedA
         results,
     };
 }
-export function runBenchmark(sourcePath, spec, targets) {
-    if (spec.cases.some((benchmarkCase) => benchmarkCaseNeedsExecutor(benchmarkCase))) {
-        throw new Error("This benchmark needs a live executor. Use runBenchmarkWithJudge instead.");
+export function runTargetTests(sourcePath, spec, targets) {
+    if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
+        throw new Error("This test needs a live executor. Use runTargetTestsWithJudge instead.");
     }
     for (const target of targets) {
         if (target.type !== spec.type) {
-            throw new Error(`Benchmark target type mismatch: expected ${spec.type}, got ${target.type}`);
+            throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
         }
     }
-    const results = targets.map((target) => buildBenchmarkTargetResult(target, spec.cases.map((benchmarkCase) => runBenchmarkCase(target, benchmarkCase))));
-    return buildBenchmarkRunResult(sourcePath, spec, results);
+    const results = targets.map((target) => buildTestTargetResult(target, spec.cases.map((testCase) => runTestCase(target, testCase))));
+    return buildTestTargetRun(sourcePath, spec, results);
 }
-export async function runBenchmarkWithJudge(sourcePath, spec, targets, executor, options = {}) {
+export async function runTargetTestsWithJudge(sourcePath, spec, targets, executor, options = {}) {
     const preserveMode = options.preserveSandboxes ?? "on-failure";
+    const artifactRootPath = options.artifactRootPath ?? sourcePath;
     const generatedAt = new Date().toISOString();
     const sandboxRunId = `${generatedAt.replace(/[:.]/g, "-")}-${spec.id}`;
     for (const target of targets) {
         if (target.type !== spec.type) {
-            throw new Error(`Benchmark target type mismatch: expected ${spec.type}, got ${target.type}`);
+            throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
         }
     }
     const results = [];
     for (const [index, target] of targets.entries()) {
-        const sandbox = createBenchmarkSandbox(target);
+        const sandbox = createTestSandbox(target);
         try {
             const sandboxTarget = {
                 ...target,
                 path: sandbox.targetPath,
             };
             const caseResults = [];
-            for (const benchmarkCase of spec.cases) {
-                if (benchmarkCaseNeedsExecutor(benchmarkCase)) {
-                    caseResults.push(await runLiveBenchmarkCase(sandboxTarget, benchmarkCase, executor));
+            for (const testCase of spec.cases) {
+                if (testCaseNeedsExecutor(testCase)) {
+                    caseResults.push(await runLiveTestCase(sandboxTarget, testCase, executor));
                 }
                 else {
-                    caseResults.push(await runBenchmarkCaseWithJudge(sandboxTarget, benchmarkCase, executor));
+                    caseResults.push(await runTestCaseWithJudge(sandboxTarget, testCase, executor));
                 }
             }
             let sandboxPath;
-            const targetResult = buildBenchmarkTargetResult(target, caseResults);
+            const targetResult = buildTestTargetResult(target, caseResults);
             const shouldPreserveSandbox = preserveMode === "always" || !targetResult.ok;
             if (shouldPreserveSandbox) {
-                const sandboxRoot = benchmarkSandboxesPath(sourcePath, target.type);
+                const sandboxRoot = targetTestSandboxesPath(artifactRootPath, target.type);
                 mkdirSync(sandboxRoot, { recursive: true });
-                const gitignorePath = benchmarkSandboxGitignorePath(sourcePath, target.type);
+                const gitignorePath = targetTestSandboxGitignorePath(artifactRootPath, target.type);
                 if (!existsSync(gitignorePath)) {
                     writeFileSync(gitignorePath, "*\n!.gitignore\n");
                 }
-                const sandboxPathName = `${String(index + 1).padStart(2, "0")}-${normalizeBenchmarkId(target.name) || target.type}`;
+                const sandboxPathName = `${String(index + 1).padStart(2, "0")}-${normalizeTestId(target.name) || target.type}`;
                 sandbox.preserve(join(sandboxRoot, sandboxRunId, sandboxPathName));
                 sandboxPath = sandbox.targetPath;
             }
-            results.push(buildBenchmarkTargetResult(target, caseResults, { sandboxPath }));
+            results.push(buildTestTargetResult(target, caseResults, { sandboxPath }));
         }
         finally {
             sandbox.cleanup();
         }
     }
-    return buildBenchmarkRunResult(sourcePath, spec, results, executor, generatedAt);
+    return buildTestTargetRun(sourcePath, spec, results, executor, generatedAt);
 }
-export async function runBenchmarkAuto(sourcePath, spec, targets, options) {
-    if (spec.cases.some((benchmarkCase) => benchmarkCaseNeedsExecutor(benchmarkCase))) {
+export async function runTargetTestsAuto(sourcePath, spec, targets, options) {
+    if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
         if (!options?.executor) {
-            throw new Error("This benchmark needs a live local executor, but no executor was provided.");
+            throw new Error("This test needs a live local executor, but no executor was provided.");
         }
-        return runBenchmarkWithJudge(sourcePath, spec, targets, options.executor, {
+        return runTargetTestsWithJudge(sourcePath, spec, targets, options.executor, {
             preserveSandboxes: options.preserveSandboxes,
+            artifactRootPath: options.artifactRootPath,
         });
     }
-    return runBenchmark(sourcePath, spec, targets);
+    return runTargetTests(sourcePath, spec, targets);
 }
-export function saveBenchmarkRun(sourcePath, result) {
-    const dirPath = benchmarkRunsPath(sourcePath, result.benchmark.type);
+export function saveTargetTestRun(artifactRootPath, result) {
+    const dirPath = targetTestRunsPath(artifactRootPath, result.spec.type);
     mkdirSync(dirPath, { recursive: true });
-    const gitignorePath = benchmarkRunGitignorePath(sourcePath, result.benchmark.type);
+    const gitignorePath = targetTestRunGitignorePath(artifactRootPath, result.spec.type);
     if (!existsSync(gitignorePath)) {
         writeFileSync(gitignorePath, "*\n!.gitignore\n");
     }
     const timestamp = result.generated_at.replace(/[:.]/g, "-");
-    const runDirPath = join(dirPath, `${timestamp}-${result.benchmark.id}`);
+    const runDirPath = join(dirPath, `${timestamp}-${result.spec.id}`);
     mkdirSync(runDirPath, { recursive: true });
     const manifestPath = join(runDirPath, "manifest.json");
     writeFileSync(manifestPath, `${JSON.stringify({
-        kind: "interf-benchmark-run-manifest",
+        kind: "interf-test-target-run-manifest",
         version: 1,
         generated_at: result.generated_at,
-        benchmark: {
-            id: result.benchmark.id,
-            name: result.benchmark.name,
-            type: result.benchmark.type,
-            case_count: result.benchmark.case_count,
+        spec: {
+            id: result.spec.id,
+            name: result.spec.name,
+            type: result.spec.type,
+            case_count: result.spec.case_count,
         },
         result_file: "run.json",
         target_count: result.target_count,
@@ -520,4 +523,3 @@ export function saveBenchmarkRun(sourcePath, result) {
     writeFileSync(runPath, `${JSON.stringify(result, null, 2)}\n`);
     return runPath;
 }
-//# sourceMappingURL=benchmark-execution.js.map

package/dist/lib/test-matrices.d.ts

@@ -0,0 +1,90 @@
+import { z } from "zod";
+import type { WorkflowExecutionProfile } from "./executors.js";
+export declare const TestMatrixAgentSchema: z.ZodEnum<{
+    "claude-code": "claude-code";
+    codex: "codex";
+}>;
+export declare const TestMatrixProfileSchema: z.ZodPipe<z.ZodObject<{
+    id: z.ZodString;
+    preset: z.ZodOptional<z.ZodEnum<{
+        "release-claude-high": "release-claude-high";
+        "release-codex-high": "release-codex-high";
+        "tier1-max-claude": "tier1-max-claude";
+        "tier1-max-codex": "tier1-max-codex";
+    }>>;
+    agent: z.ZodOptional<z.ZodEnum<{
+        "claude-code": "claude-code";
+        codex: "codex";
+    }>>;
+    model: z.ZodOptional<z.ZodString>;
+    profile: z.ZodOptional<z.ZodString>;
+    effort: z.ZodOptional<z.ZodString>;
+    timeout_ms: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodTransform<{
+    agent: "claude-code" | "codex";
+    id: string;
+    preset?: "release-claude-high" | "release-codex-high" | "tier1-max-claude" | "tier1-max-codex" | undefined;
+    model?: string | undefined;
+    profile?: string | undefined;
+    effort?: string | undefined;
+    timeout_ms?: number | undefined;
+}, {
+    id: string;
+    preset?: "release-claude-high" | "release-codex-high" | "tier1-max-claude" | "tier1-max-codex" | undefined;
+    agent?: "claude-code" | "codex" | undefined;
+    model?: string | undefined;
+    profile?: string | undefined;
+    effort?: string | undefined;
+    timeout_ms?: number | undefined;
+}>>;
+export declare const TestMatrixRetryPolicySchema: z.ZodObject<{
+    max_attempts_per_profile: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const TestMatrixWorkspaceSchema: z.ZodString;
+export declare const TestMatrixSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    source_path: z.ZodString;
+    compile_profiles: z.ZodArray<z.ZodPipe<z.ZodObject<{
+        id: z.ZodString;
+        preset: z.ZodOptional<z.ZodEnum<{
+            "release-claude-high": "release-claude-high";
+            "release-codex-high": "release-codex-high";
+            "tier1-max-claude": "tier1-max-claude";
+            "tier1-max-codex": "tier1-max-codex";
+        }>>;
+        agent: z.ZodOptional<z.ZodEnum<{
+            "claude-code": "claude-code";
+            codex: "codex";
+        }>>;
+        model: z.ZodOptional<z.ZodString>;
+        profile: z.ZodOptional<z.ZodString>;
+        effort: z.ZodOptional<z.ZodString>;
+        timeout_ms: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodTransform<{
+        agent: "claude-code" | "codex";
+        id: string;
+        preset?: "release-claude-high" | "release-codex-high" | "tier1-max-claude" | "tier1-max-codex" | undefined;
+        model?: string | undefined;
+        profile?: string | undefined;
+        effort?: string | undefined;
+        timeout_ms?: number | undefined;
+    }, {
+        id: string;
+        preset?: "release-claude-high" | "release-codex-high" | "tier1-max-claude" | "tier1-max-codex" | undefined;
+        agent?: "claude-code" | "codex" | undefined;
+        model?: string | undefined;
+        profile?: string | undefined;
+        effort?: string | undefined;
+        timeout_ms?: number | undefined;
+    }>>>;
+    retry_policy: z.ZodOptional<z.ZodObject<{
+        max_attempts_per_profile: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    workspaces: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export type TestMatrixProfile = z.infer<typeof TestMatrixProfileSchema>;
+export type TestMatrixRetryPolicy = z.infer<typeof TestMatrixRetryPolicySchema>;
+export type TestMatrixWorkspace = z.infer<typeof TestMatrixWorkspaceSchema>;
+export type TestMatrix = z.infer<typeof TestMatrixSchema>;
+export declare function testMatrixProfileToExecutionProfile(profile: TestMatrixProfile): WorkflowExecutionProfile;