@interf/compiler 0.22.2 → 0.50.0

package/dist/packages/{engine → runtime}/verify/lib/schema.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
-import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, PreparationNameSchema, } from "../../../contracts/lib/schema.js";
+import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, ProjectIdSchema, } from "../../../contracts/lib/schema.js";
+import { isSafeRelativeTestFile } from "../test-file-guard.js";
 const TestCaseCoreSchema = z.object({
     id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
     question: z.string().min(1),
@@ -14,6 +15,17 @@ const TestCaseCoreSchema = z.object({
             message: "Test cases need at least one of file, answer, or expect.",
         });
     }
+    // H2: a `file` target is a project-relative output path that is later joined
+    // onto the (sandboxed) target directory. Reject absolute or `..`-escaping
+    // values at parse time so they can never reach `path.join`. Centralizing the
+    // guard here means H1 (verify-execution join) and H3 (spec writer) inherit it.
+    if (value.file !== undefined && !isSafeRelativeTestFile(value.file)) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ["file"],
+            message: "Test case file must be a relative path without '..' or a leading '/'.",
+        });
+    }
 });
 export const TestCaseSchema = TestCaseCoreSchema;
 const TestSpecCoreSchema = z.object({
@@ -81,7 +93,13 @@ export const TestTargetRunSchema = z.object({
     target_count: z.number(),
     results: z.array(TestTargetResultSchema),
 });
-export const TestRunModeSchema = z.enum(["source-files", "verifiable-context", "both"]);
+export const TestRunModeSchema = z.enum(["source-files", "context-graph", "both"]);
+export const BenchmarkSourceUsageSchema = z.enum([
+    "source-baseline",
+    "graph-only",
+    "graph-source-verified",
+    "unknown",
+]);
 export const TestRunTargetSummarySchema = z.object({
     label: z.string(),
     run_path: z.string(),
@@ -90,6 +108,8 @@ export const TestRunTargetSummarySchema = z.object({
     total_cases: z.number(),
     passed_checks: z.number(),
     total_checks: z.number(),
+    executor: RuntimeExecutorInfoSchema.nullable().optional(),
+    source_usage: BenchmarkSourceUsageSchema.default("unknown"),
     target: z.object({
         type: TestTargetTypeSchema,
         name: z.string(),
@@ -97,12 +117,12 @@ export const TestRunTargetSummarySchema = z.object({
         build_plan: z.string(),
     }),
 });
-function mapReadinessCheckRun(value) {
+function mapBenchmarkRun(value) {
     if (!value || typeof value !== "object" || Array.isArray(value))
         return value;
     const record = value;
-    const preparation = record.preparation && typeof record.preparation === "object" && !Array.isArray(record.preparation)
-        ? record.preparation
+    const project = record.project && typeof record.project === "object" && !Array.isArray(record.project)
+        ? record.project
         : null;
     const summary = record.summary && typeof record.summary === "object" && !Array.isArray(record.summary)
         ? record.summary
@@ -110,36 +130,36 @@ function mapReadinessCheckRun(value) {
     return {
         ...record,
         mode: record.mode,
-        preparation: preparation
+        project: project
             ? {
-                ...preparation,
-                verifiable_context_path: preparation.verifiable_context_path,
+                ...project,
+                context_graph_path: project.context_graph_path,
             }
-            : record.preparation,
-        verifiable_context: record.verifiable_context,
+            : record.project,
+        context_graph: record.context_graph,
         summary: summary
             ? {
                 ...summary,
-                verifiable_context_pass_rate: summary.verifiable_context_pass_rate,
+                context_graph_pass_rate: summary.context_graph_pass_rate,
             }
             : record.summary,
     };
 }
-export const ReadinessCheckRunSchema = z.preprocess(mapReadinessCheckRun, z.object({
-    kind: z.literal("interf-readiness-check-run"),
+export const BenchmarkRunSchema = z.preprocess(mapBenchmarkRun, z.object({
+    kind: z.literal("interf-benchmark-run"),
     version: z.literal(1),
     generated_at: z.string(),
     mode: TestRunModeSchema,
     source_path: z.string(),
     checks_fingerprint: z.string().min(1).optional(),
-    preparation: z.object({
-        name: PreparationNameSchema,
-        verifiable_context_path: z.string().nullable(),
+    project: z.object({
+        name: ProjectIdSchema,
+        context_graph_path: z.string().nullable(),
     }),
     source_files: TestRunTargetSummarySchema.nullable(),
-    verifiable_context: TestRunTargetSummarySchema.nullable(),
+    context_graph: TestRunTargetSummarySchema.nullable(),
     summary: z.object({
         source_files_pass_rate: z.number().nullable(),
-        verifiable_context_pass_rate: z.number().nullable(),
+        context_graph_pass_rate: z.number().nullable(),
     }),
 }));

package/dist/packages/runtime/verify/test-file-guard.d.ts

@@ -0,0 +1,2 @@
+export declare function isSafeRelativeTestFile(file: string): boolean;
+export declare function assertSafeRelativeTestFile(file: string): void;

package/dist/packages/runtime/verify/test-file-guard.js

@@ -0,0 +1,29 @@
+// Pure, Node-free guard for a `testCase.file` value. A test file is a
+// project-relative output path inside the (sandboxed) target directory, so it
+// must never be absolute, never escape via `..`, and stay within a conservative
+// character set. This is the single guard both the schema (H2) and the spec
+// writer (H3) consume so a malicious `file` cannot reach `path.join` and a later
+// `existsSync`/`readFileSync`.
+//
+// This module imports no Node builtins so it is safe to pull into the schemas
+// barrel (`@interf/compiler/schemas`) and bundle into the renderer. The
+// filesystem-aware path helpers live in `verify-paths.ts`, which re-exports
+// these guards for server-side callers.
+const TEST_FILE_PATTERN = /^[A-Za-z0-9._/-]+$/;
+export function isSafeRelativeTestFile(file) {
+    if (file.length === 0)
+        return false;
+    if (file.startsWith("/"))
+        return false;
+    if (!TEST_FILE_PATTERN.test(file))
+        return false;
+    // Reject `..` as a whole path segment (e.g. `..`, `../x`, `a/../b`, `a/..`).
+    if (file.split("/").some((segment) => segment === ".."))
+        return false;
+    return true;
+}
+export function assertSafeRelativeTestFile(file) {
+    if (!isSafeRelativeTestFile(file)) {
+        throw new Error(`Invalid test case file: ${file}`);
+    }
+}

package/dist/packages/{engine → runtime}/verify/verify-execution.d.ts

@@ -1,6 +1,12 @@
 import { type AgentExecutor } from "../agents/lib/executors.js";
 import type { TestTargetRun, TestTargetCandidate, LoadedTestSpec } from "./verify-types.js";
 import { type TestSandboxRetentionMode } from "./verify-sandbox.js";
+export declare function resolveTargetFilePath(targetPath: string, file: string): string;
+interface TestJudgeVerdict {
+    pass: boolean;
+    summary: string;
+}
+export declare function readTestJudgeVerdictFromStatus(statusPath: string): TestJudgeVerdict | null;
 export declare function runTargetTests(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[]): TestTargetRun;
 export declare function runTargetTestsWithJudge(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], executor: AgentExecutor, options?: {
     preserveSandboxes?: TestSandboxRetentionMode;
@@ -12,3 +18,4 @@ export declare function runTargetTestsAuto(sourcePath: string, spec: LoadedTestS
     artifactRootPath?: string;
 }): Promise<TestTargetRun>;
 export declare function saveTargetTestRun(artifactRootPath: string, result: TestTargetRun): string;
+export {};

package/dist/packages/{engine → runtime}/verify/verify-execution.js

@@ -1,9 +1,21 @@
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
+import { assertPathWithinRoot } from "../../contracts/utils/path-guards.js";
 import { buildRuntimeExecutorInfo } from "../agents/lib/executors.js";
 import { targetTestRunGitignorePath, targetTestRunsPath, targetTestSandboxGitignorePath, targetTestSandboxesPath, normalizeTestId, } from "./verify-paths.js";
 import { createTestSandbox, } from "./verify-sandbox.js";
+import { freezePreservedShell } from "../agents/lib/shell-fs.js";
+// H1: `testCase.file` is a project-relative output path joined onto the target
+// directory. `path.join` resolves `..` at string level, so a malicious file can
+// escape the (sandboxed) target and be read via existsSync/readFileSync. Resolve
+// the join and assert it stays within target.path before any filesystem access.
+// Centralized so every read of a test-case file inherits the bound (ENGINE-POLICY
+// rule 7). The schema/spec-writer guards (H2/H3) are defense-in-depth ahead of
+// this; this is the last line before the filesystem.
+export function resolveTargetFilePath(targetPath, file) {
+    return assertPathWithinRoot(targetPath, join(targetPath, file), "test case file");
+}
 function parseWords(content) {
     return content.trim().split(/\s+/).filter(Boolean).length;
 }
@@ -66,7 +78,7 @@ function runTestCase(target, testCase) {
     if (!testCase.file) {
         throw new Error(`Test case "${testCase.id}" requires an executor because it has no file target.`);
     }
-    const outputPath = join(target.path, testCase.file);
+    const outputPath = resolveTargetFilePath(target.path, testCase.file);
     const checks = [];
     if (!existsSync(outputPath)) {
         checks.push({
@@ -133,7 +145,13 @@ function readTestJudgeVerdict(verdictPath) {
         summary: typeof raw.summary === "string" ? raw.summary : "",
     };
 }
-function readTestJudgeVerdictFromStatus(statusPath) {
+// M13: verdict.json is the single source of truth for a judge pass/fail. This is
+// a strict fallback for when the agent omitted the JSON file: only an explicit
+// `pass=true` / `pass=false` token on a terminal DONE:/BLOCKED:/ERROR: line is
+// honored. The prompt mandates `DONE: pass=true|false - <summary>`, so a benign
+// status line that merely mentions a match must NOT be read as a pass. A terminal
+// line with no explicit pass token yields no verdict (treated as missing/invalid).
+export function readTestJudgeVerdictFromStatus(statusPath) {
     if (!existsSync(statusPath))
         return null;
     const lines = readFileSync(statusPath, "utf8")
@@ -148,79 +166,127 @@ function readTestJudgeVerdictFromStatus(statusPath) {
             continue;
         const normalized = line.toLowerCase();
         const summary = line.replace(/^(DONE|BLOCKED|ERROR):\s*/i, "").trim();
-        if (normalized.includes("pass=true")) {
-            return { pass: true, summary };
-        }
-        if (normalized.includes("pass=false")) {
-            return { pass: false, summary };
-        }
-        if (/values match expected|matches expected|candidate matches|answer matches|expected values match/i.test(line)) {
+        if (/\bpass=true\b/.test(normalized)) {
             return { pass: true, summary };
         }
-        if (/does not match|do not match|mismatch|candidate fails|answer fails|expected values do not match/i.test(line)) {
+        if (/\bpass=false\b/.test(normalized)) {
             return { pass: false, summary };
         }
+        // Terminal line reached without an explicit pass token: no trustworthy
+        // verdict. Stop scanning so an earlier, benign line cannot be misread.
+        return null;
     }
     return null;
 }
-async function runTargetTestsJudge(testCase, executor, candidateLabel, candidateContent) {
-    const tempDir = mkdtempSync(join(tmpdir(), "interf-test-judge-"));
+async function runTargetTestsJudge(testCase, executor, candidateLabel, candidateContent, 
+// When supplied, the judge shell is created at this durable path and preserved
+// (frozen) on finish so the judge execution is inspectable: prompt, reasoning
+// transcript, the candidate it judged, and the JSON verdict all survive. Lives
+// under the benchmark sandbox so it is preserved with the sandbox on failure.
+// When omitted, the shell is an ephemeral /tmp dir removed on finish.
+preservedShellRoot) {
+    const preserve = Boolean(preservedShellRoot);
+    const tempDir = preservedShellRoot
+        ? (mkdirSync(preservedShellRoot, { recursive: true }), preservedShellRoot)
+        : mkdtempSync(join(tmpdir(), "interf-test-judge-"));
+    // The preserved shell needs a runtime/ dir up front: freezePreservedShell
+    // writes its manifest to runtime/preserved-shell.json, and reasoning is teed
+    // there too. (The ephemeral path doesn't need it.)
+    if (preserve) {
+        mkdirSync(join(tempDir, "runtime"), { recursive: true });
+    }
     let executionError = null;
     let verdict = null;
     try {
         const verdictPath = join(tempDir, "verdict.json");
         const statusPath = join(tempDir, "judge.status.log");
         const prompt = buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath);
+        // Preserve the rendered prompt and the candidate it judged so the preserved
+        // shell holds everything needed to reproduce the verdict, not just the output.
+        if (preserve) {
+            writeFileSync(join(tempDir, "prompt.txt"), `${prompt}\n`);
+            writeFileSync(join(tempDir, "candidate.txt"), `${candidateContent}\n`);
+        }
+        // Same canonical convention as the stage path: reasoning is teed into the
+        // shell's runtime/ dir so it is preserved when the shell is frozen.
+        const reasoningLogPath = preserve ? join(tempDir, "runtime", "agent-reasoning.jsonl") : null;
         try {
             await executor.execute(tempDir, prompt, {
                 statusLogPath: statusPath,
+                reasoningLogPath,
             });
         }
         catch (error) {
             executionError = error instanceof Error ? error.message : String(error);
         }
-        try {
-            verdict = readTestJudgeVerdict(verdictPath);
-            if (!verdict) {
-                verdict = readTestJudgeVerdictFromStatus(statusPath);
+        // M12: first error wins. If execute() already failed, do not read verdict
+        // files — a throw in verdict-reading would otherwise overwrite the real
+        // execution failure and mask why the judge never ran. Only read the verdict
+        // when execution itself reported no error.
+        if (!executionError) {
+            try {
+                verdict = readTestJudgeVerdict(verdictPath);
+                if (!verdict) {
+                    verdict = readTestJudgeVerdictFromStatus(statusPath);
+                }
+            }
+            catch (error) {
+                executionError = error instanceof Error ? error.message : String(error);
             }
-        }
-        catch (error) {
-            executionError = error instanceof Error ? error.message : String(error);
         }
     }
     finally {
-        rmSync(tempDir, { recursive: true, force: true });
+        // Preserve a durable judge shell (freeze materializes symlinks + writes the
+        // preserved-shell manifest, path unchanged); only remove an ephemeral one.
+        if (preserve) {
+            freezePreservedShell(tempDir, "test-judge");
+        }
+        else {
+            rmSync(tempDir, { recursive: true, force: true });
+        }
     }
     return { verdict, error: executionError };
 }
-function buildTestQueryPrompt(target, testCase, answerPath, tracePath) {
-    const header = target.type === "verifiable-context"
+/**
+ * Durable judge-shell root under a benchmark sandbox target. Lives beside the
+ * sandbox's other preserved runtime artifacts so a preserved sandbox carries the
+ * judge execution too. Keyed by case + candidate so multiple judged cases in one
+ * target don't collide.
+ */
+function judgeShellRoot(targetPath, testCase, candidateLabel) {
+    const slug = `${normalizeTestId(testCase.id) || "case"}-${normalizeTestId(candidateLabel) || "candidate"}`;
+    return join(targetPath, ".interf", "runtime", "test-judge", slug);
+}
+function buildTestQueryPrompt(target, testCase, answerPath, tracePath, retryReason) {
+    const header = target.type === "context-graph"
         ? [
-            "You are running an Interf test inside an isolated sandboxed verifiable-context copy.",
+            "You are running an Interf benchmark inside an isolated sandboxed context-graph copy.",
             "Read `AGENTS.md` first.",
-            "Use the local native `interf-query` skill available in this verifiable context.",
-            "Answer the check question the same way you would answer a real user inside this verifiable context.",
-            "Prefer the Build Plan-declared verifiable-context outputs before consulting source references.",
-            "This sandbox is self-contained: the copied verifiable context has source references in `.interf/runtime/source-snapshot.json`.",
-            "The original Interf instance state is intentionally absent from this sandbox. Use the source locators only when the verifiable context needs verification.",
+            "Use the local native `interf-query` skill available in this Context Graph.",
+            "Answer the check question the same way you would answer a real user inside this Context Graph.",
+            "Prefer the Build Plan-declared context-graph outputs before consulting source references.",
+            "This sandbox is self-contained: the copied Context Graph has authoritative source references in `.interf/runtime/source-manifest.json`.",
+            "The original Interf instance state is intentionally absent from this sandbox. Use the source locators only when the Context Graph needs verification.",
         ]
         : [
-            "You are running an Interf baseline test inside an isolated source-files test shell.",
+            "You are running an Interf baseline benchmark inside an isolated source-files benchmark shell.",
             "Read `AGENTS.md` first.",
             "Use the local native `interf-query` skill available in this shell.",
-            "There is no verifiable context in this sandbox.",
-            "Answer only from source references listed in `runtime/source-files.json` inside this shell.",
+            "There is no Context Graph in this sandbox.",
+            "Answer only from source references listed in `runtime/source-manifest.json` inside this shell.",
             "The original control plane folder is intentionally absent from this sandbox.",
         ];
     return [
         ...header,
         "Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
         "Do not ask follow-up questions.",
+        "Create both required output files before printing DONE.",
+        "The output paths are absolute temp file paths outside the Context Graph sandbox; write them exactly as given.",
         `Write the answer to ${JSON.stringify(answerPath)}.`,
         `Write the trace to ${JSON.stringify(tracePath)} with keys: case_id, target, artifacts_consulted, source_paths_read, used_source_files, answer_summary.`,
         `Set \`case_id\` to ${JSON.stringify(testCase.id)}.`,
         `Set \`target\` to ${JSON.stringify(target.type)}.`,
+        ...(retryReason ? [`Retry reason: ${retryReason}`] : []),
         `Question: ${testCase.question}`,
     ].join("\n");
 }
@@ -228,20 +294,26 @@ async function runLiveTestCase(target, testCase, executor) {
     const tempDir = mkdtempSync(join(tmpdir(), "interf-test-live-"));
     const answerPath = join(tempDir, "answer.md");
     const tracePath = join(tempDir, "trace.json");
-    const statusPath = join(tempDir, "status.log");
-    const eventPath = join(tempDir, "events.ndjson");
-    const prompt = buildTestQueryPrompt(target, testCase, answerPath, tracePath);
     let executionError = null;
     let code = -1;
     try {
-        try {
-            code = await executor.execute(target.path, prompt, {
-                eventLogPath: eventPath,
-                statusLogPath: statusPath,
-            });
-        }
-        catch (error) {
-            executionError = error instanceof Error ? error.message : String(error);
+        for (let attempt = 1; attempt <= 2; attempt += 1) {
+            const statusPath = join(tempDir, `status-${attempt}.log`);
+            const eventPath = join(tempDir, `events-${attempt}.ndjson`);
+            const prompt = buildTestQueryPrompt(target, testCase, answerPath, tracePath, attempt === 1
+                ? undefined
+                : `Attempt ${attempt - 1} exited without writing ${!existsSync(answerPath) ? "the answer file" : "the trace file"}. Write both files before DONE.`);
+            try {
+                code = await executor.execute(target.path, prompt, {
+                    eventLogPath: eventPath,
+                    statusLogPath: statusPath,
+                });
+            }
+            catch (error) {
+                executionError = error instanceof Error ? error.message : String(error);
+            }
+            if (existsSync(answerPath) && existsSync(tracePath))
+                break;
         }
         const checks = [];
         if (!existsSync(answerPath)) {
@@ -294,7 +366,8 @@ async function runLiveTestCase(target, testCase, executor) {
             });
         }
         if (testCase.answer) {
-            const judged = await runTargetTestsJudge(testCase, executor, `generated answer for ${testCase.id}`, answer);
+            const candidateLabel = `generated answer for ${testCase.id}`;
+            const judged = await runTargetTestsJudge(testCase, executor, candidateLabel, answer, judgeShellRoot(target.path, testCase, candidateLabel));
             checks.push({
                 label: "judge verdict",
                 ok: judged.verdict?.pass === true,
@@ -324,7 +397,7 @@ async function runTestCaseWithJudge(target, testCase, executor) {
     if (!testCase.file) {
         return runLiveTestCase(target, testCase, executor);
     }
-    const outputPath = join(target.path, testCase.file);
+    const outputPath = resolveTargetFilePath(target.path, testCase.file);
     const checks = [];
     if (!existsSync(outputPath)) {
         checks.push({
@@ -353,7 +426,8 @@ async function runTestCaseWithJudge(target, testCase, executor) {
     });
     checks.push(...evaluated.checks);
     if (testCase.answer) {
-        const judged = await runTargetTestsJudge(testCase, executor, `verifiable context file ${outputPath}`, content);
+        const candidateLabel = `Context Graph file ${outputPath}`;
+        const judged = await runTargetTestsJudge(testCase, executor, candidateLabel, content, judgeShellRoot(target.path, testCase, candidateLabel));
         checks.push({
             label: "judge verdict",
             ok: judged.verdict?.pass === true,

package/dist/packages/{engine → runtime}/verify/verify-paths.d.ts

@@ -1,11 +1,12 @@
 import type { TestTargetType } from "./verify-types.js";
 export declare const TEST_SPEC_EXTENSIONS: Set<string>;
+export { isSafeRelativeTestFile, assertSafeRelativeTestFile, } from "./test-file-guard.js";
 export declare function testSpecRootPath(sourcePath: string): string;
 export declare function testSpecTypePath(sourcePath: string, type: TestTargetType): string;
-export declare function targetTestRunsPath(verifiableContextPath: string, type: TestTargetType): string;
-export declare function targetTestRunGitignorePath(verifiableContextPath: string, type: TestTargetType): string;
-export declare function targetTestSandboxesPath(verifiableContextPath: string, type: TestTargetType): string;
-export declare function targetTestSandboxGitignorePath(verifiableContextPath: string, type: TestTargetType): string;
+export declare function targetTestRunsPath(contextGraphPath: string, type: TestTargetType): string;
+export declare function targetTestRunGitignorePath(contextGraphPath: string, type: TestTargetType): string;
+export declare function targetTestSandboxesPath(contextGraphPath: string, type: TestTargetType): string;
+export declare function targetTestSandboxGitignorePath(contextGraphPath: string, type: TestTargetType): string;
 export declare function testSpecFilePath(sourcePath: string, type: TestTargetType, id: string): string;
 export declare function normalizeTestId(input: string): string;
 export declare function assertTestId(id: string): void;

package/dist/packages/runtime/verify/verify-paths.js

@@ -0,0 +1,65 @@
+import { existsSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { targetTestRunsRootForContextGraph, targetTestSandboxesRootForContextGraph, contextGraphInterfConfigPath, } from "../build/context-graph-paths.js";
+import { asProjectDataDir, projectTestsSpecsRoot, } from "../../contracts/lib/project-paths.js";
+const TEST_ID_PATTERN = /^[a-z0-9][a-z0-9-]{0,79}$/;
+export const TEST_SPEC_EXTENSIONS = new Set([".json"]);
+// The `testCase.file` safelist guard lives in a Node-free module so the schemas
+// barrel can import it without pulling Node builtins into the renderer bundle.
+// Re-exported here for server-side path callers.
+export { isSafeRelativeTestFile, assertSafeRelativeTestFile, } from "./test-file-guard.js";
+export function testSpecRootPath(sourcePath) {
+    return projectTestsSpecsRoot(asProjectDataDir(sourcePath));
+}
+export function testSpecTypePath(sourcePath, type) {
+    return join(testSpecRootPath(sourcePath), type);
+}
+function isProjectTestsRootPath(artifactRootPath) {
+    return basename(artifactRootPath) === "tests" &&
+        basename(dirname(artifactRootPath)) === ".interf";
+}
+export function targetTestRunsPath(contextGraphPath, type) {
+    if (existsSync(contextGraphInterfConfigPath(contextGraphPath))) {
+        return join(targetTestRunsRootForContextGraph(contextGraphPath), type);
+    }
+    if (isProjectTestsRootPath(contextGraphPath)) {
+        return join(contextGraphPath, type === "source-files" ? "source-files" : "context-graph", "runs");
+    }
+    throw new Error(`Unsupported test artifact root: ${contextGraphPath}. Expected context graph or interf/<project>/.interf/tests root.`);
+}
+export function targetTestRunGitignorePath(contextGraphPath, type) {
+    return join(targetTestRunsPath(contextGraphPath, type), ".gitignore");
+}
+export function targetTestSandboxesPath(contextGraphPath, type) {
+    if (existsSync(contextGraphInterfConfigPath(contextGraphPath))) {
+        return join(targetTestSandboxesRootForContextGraph(contextGraphPath), type);
+    }
+    if (isProjectTestsRootPath(contextGraphPath)) {
+        return join(contextGraphPath, type === "source-files" ? "source-files" : "context-graph", "sandboxes");
+    }
+    throw new Error(`Unsupported test artifact root: ${contextGraphPath}. Expected context graph or interf/<project>/.interf/tests root.`);
+}
+export function targetTestSandboxGitignorePath(contextGraphPath, type) {
+    return join(targetTestSandboxesPath(contextGraphPath, type), ".gitignore");
+}
+export function testSpecFilePath(sourcePath, type, id) {
+    return join(testSpecTypePath(sourcePath, type), `${id}.json`);
+}
+export function normalizeTestId(input) {
+    return input
+        .toLowerCase()
+        .trim()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 80);
+}
+export function assertTestId(id) {
+    if (!TEST_ID_PATTERN.test(id)) {
+        throw new Error(`Invalid test id: ${id}`);
+    }
+}
+export function assertWritableTestSpecPath(filePath) {
+    if (existsSync(filePath)) {
+        throw new Error(`Test spec already exists: ${filePath}`);
+    }
+}

package/dist/packages/{engine → runtime}/verify/verify-sandbox.d.ts

@@ -3,7 +3,7 @@ export type TestSandboxRetentionMode = "on-failure" | "always";
 export interface TestSandbox {
     rootPath: string;
     targetPath: string;
-    verifiableContextPath: string | null;
+    contextGraphPath: string | null;
     preserve(destinationPath: string): string;
     cleanup(): void;
 }

package/dist/packages/runtime/verify/verify-sandbox.js

@@ -0,0 +1,88 @@
+import { cpSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { refreshContextGraphBootstrapGuidance } from "../agents/lib/context-graph-bootstrap.js";
+import { readInterfConfig } from "../../projects/interf.js";
+import { projectSourceFilesTestQueryShell } from "../agents/lib/shells.js";
+import { asProjectDataDir, projectContextGraphPath, } from "../../contracts/lib/project-paths.js";
+import { saveContextGraphInterfConfig } from "../../projects/source-config.js";
+import { loadContextGraphSourceManifest, sourceManifestToSourceFiles, } from "../build/source-manifest.js";
+import { refreshContextGraphArtifacts } from "../build/state.js";
+import { testRootForContextGraph, stageExecutionShellsRoot, targetTestSandboxesRootForContextGraph, buildPlanImprovementLoopsRoot, contextGraphQueryAcceptanceRoot, contextGraphRuntimeLogsRoot, } from "../build/context-graph-paths.js";
+function sanitizeContextGraphArtifacts(contextGraphPath) {
+    for (const absolutePath of [
+        testRootForContextGraph(contextGraphPath),
+        targetTestSandboxesRootForContextGraph(contextGraphPath),
+        stageExecutionShellsRoot(contextGraphPath),
+        buildPlanImprovementLoopsRoot(contextGraphPath),
+        contextGraphRuntimeLogsRoot(contextGraphPath),
+        contextGraphQueryAcceptanceRoot(contextGraphPath),
+    ]) {
+        rmSync(absolutePath, { recursive: true, force: true });
+    }
+}
+export function createTestSandbox(target) {
+    let sandboxRoot = mkdtempSync(join(tmpdir(), "interf-test-sandbox-"));
+    let preserved = false;
+    const sandboxContextGraphPath = () => (target.type === "context-graph"
+        ? projectContextGraphPath(asProjectDataDir(sandboxRoot), target.name)
+        : null);
+    const sandboxTargetPath = () => (target.type === "context-graph"
+        ? projectContextGraphPath(asProjectDataDir(sandboxRoot), target.name)
+        : sandboxRoot);
+    try {
+        if (target.type === "context-graph") {
+            mkdirSync(dirname(sandboxTargetPath()), { recursive: true });
+            cpSync(target.path, sandboxTargetPath(), { recursive: true });
+            sanitizeContextGraphArtifacts(sandboxTargetPath());
+            const sandboxConfig = readInterfConfig(sandboxTargetPath());
+            if (!sandboxConfig) {
+                throw new Error(`Sandbox Context Graph is missing interf.json: ${sandboxTargetPath()}`);
+            }
+            saveContextGraphInterfConfig(sandboxTargetPath(), {
+                ...sandboxConfig,
+                source: sandboxConfig.source,
+            });
+            refreshContextGraphBootstrapGuidance(sandboxTargetPath());
+            refreshContextGraphArtifacts(sandboxTargetPath(), { ensureViewSpec: true });
+        }
+        else {
+            mkdirSync(join(sandboxTargetPath(), "runtime"), { recursive: true });
+            const manifest = loadContextGraphSourceManifest(target.path);
+            if (!manifest) {
+                throw new Error(`Source-files benchmark target has no Source Manifest: ${target.path}`);
+            }
+            writeFileSync(join(sandboxTargetPath(), "runtime", "source-manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
+            writeFileSync(join(sandboxTargetPath(), "runtime", "source-files.json"), `${JSON.stringify(sourceManifestToSourceFiles(manifest), null, 2)}\n`);
+            projectSourceFilesTestQueryShell(sandboxTargetPath());
+        }
+    }
+    catch (error) {
+        rmSync(sandboxRoot, { recursive: true, force: true });
+        throw error;
+    }
+    return {
+        get rootPath() {
+            return sandboxRoot;
+        },
+        get targetPath() {
+            return sandboxTargetPath();
+        },
+        get contextGraphPath() {
+            return sandboxContextGraphPath();
+        },
+        preserve(destinationPath) {
+            mkdirSync(dirname(destinationPath), { recursive: true });
+            rmSync(destinationPath, { recursive: true, force: true });
+            renameSync(sandboxRoot, destinationPath);
+            sandboxRoot = destinationPath;
+            preserved = true;
+            return sandboxRoot;
+        },
+        cleanup() {
+            if (preserved)
+                return;
+            rmSync(sandboxRoot, { recursive: true, force: true });
+        },
+    };
+}

package/dist/packages/{engine → runtime}/verify/verify-specs.d.ts

@@ -4,4 +4,6 @@ export declare function loadTestSpec(sourcePath: string, type: TestTargetType, i
 export declare function loadTestSpecFromFile(filePath: string): LoadedTestSpec | null;
 export declare function writeTestSpec(sourcePath: string, spec: TestSpec & {
     id?: string;
+}, options?: {
+    overwrite?: boolean;
 }): string;