@interf/compiler 0.5.1 → 0.6.3

package/dist/packages/testing/lib/schema.js

@@ -0,0 +1,119 @@
+import { z } from "zod";
+import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, } from "../../compiler/lib/schema.js";
+import { DatasetNameSchema } from "../../project-model/lib/schema.js";
+const TestCaseCoreSchema = z.object({
+    id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
+    question: z.string().min(1),
+    file: z.string().min(1).optional(),
+    answer: z.string().min(1).optional(),
+    expect: TestCaseExpectSchema.optional(),
+    strictness: z.string().min(1).optional(),
+}).superRefine((value, ctx) => {
+    if (!value.file && !value.answer && !value.expect) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            message: "Test cases need at least one of file, answer, or expect.",
+        });
+    }
+});
+export const TestCaseSchema = TestCaseCoreSchema;
+const TestSpecCoreSchema = z.object({
+    type: TestTargetTypeSchema,
+    name: z.string().min(1),
+    description: z.string().min(1).optional(),
+    cases: z.array(TestCaseSchema).min(1),
+});
+export const TestSpecSchema = TestSpecCoreSchema;
+export const TestCheckResultSchema = z.object({
+    label: z.string(),
+    ok: z.boolean(),
+    detail: z.string(),
+});
+const JsonRecordSchema = z.record(z.string(), z.unknown());
+export const TestCaseResultSchema = z.object({
+    caseId: z.string(),
+    question: z.string(),
+    file: z.string().optional(),
+    ok: z.boolean(),
+    wordCount: z.number(),
+    passedChecks: z.number(),
+    totalChecks: z.number(),
+    checks: z.array(TestCheckResultSchema),
+    answer: z.string().optional(),
+    trace: JsonRecordSchema.nullable().optional(),
+});
+export const TestTargetResultSchema = z.object({
+    target: z.object({
+        type: TestTargetTypeSchema,
+        name: z.string(),
+        path: z.string(),
+        workflow: z.string(),
+    }),
+    sandbox_path: z.string().optional(),
+    ok: z.boolean(),
+    passedCases: z.number(),
+    totalCases: z.number(),
+    passedChecks: z.number(),
+    totalChecks: z.number(),
+    caseResults: z.array(TestCaseResultSchema),
+});
+export const TestTargetRunSchema = z.object({
+    kind: z.literal("interf-test-target-run"),
+    version: z.literal(1),
+    generated_at: z.string(),
+    spec: z.object({
+        id: z.string(),
+        name: z.string(),
+        type: TestTargetTypeSchema,
+        file: z.string(),
+        description: z.string().optional(),
+        case_count: z.number(),
+        cases: z.array(z.object({
+            id: z.string(),
+            question: z.string(),
+            file: z.string().optional(),
+            answer: z.string().optional(),
+            strictness: z.string().optional(),
+            expect: TestCaseExpectSchema.optional(),
+        })),
+    }),
+    source_path: z.string(),
+    executor: RuntimeExecutorInfoSchema.optional(),
+    target_count: z.number(),
+    results: z.array(TestTargetResultSchema),
+});
+export const TestRunModeSchema = z.enum(["raw", "compiled", "both"]);
+export const TestRunTargetSummarySchema = z.object({
+    label: z.string(),
+    run_path: z.string(),
+    ok: z.boolean(),
+    passed_cases: z.number(),
+    total_cases: z.number(),
+    passed_checks: z.number(),
+    total_checks: z.number(),
+    target: z.object({
+        type: TestTargetTypeSchema,
+        name: z.string(),
+        path: z.string(),
+        workflow: z.string(),
+    }),
+});
+export const TestRunComparisonSchema = z.object({
+    kind: z.literal("interf-test-run"),
+    version: z.literal(1),
+    generated_at: z.string(),
+    mode: TestRunModeSchema,
+    source_path: z.string(),
+    checks_fingerprint: z.string().min(1).optional(),
+    dataset: z.object({
+        name: DatasetNameSchema,
+        compiled_path: z.string().nullable(),
+    }),
+    raw: TestRunTargetSummarySchema.nullable(),
+    compiled: TestRunTargetSummarySchema.nullable(),
+    summary: z.object({
+        raw_pass_rate: z.number().nullable(),
+        compiled_pass_rate: z.number().nullable(),
+        pass_rate_delta: z.number().nullable(),
+    }),
+});

package/dist/packages/testing/test-execution.d.ts

@@ -0,0 +1,14 @@
+import { type WorkflowExecutor } from "../agents/lib/executors.js";
+import type { TestTargetRun, TestTargetCandidate, LoadedTestSpec } from "./test-types.js";
+import { type TestSandboxRetentionMode } from "./test-sandbox.js";
+export declare function runTargetTests(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[]): TestTargetRun;
+export declare function runTargetTestsWithJudge(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], executor: WorkflowExecutor, options?: {
+    preserveSandboxes?: TestSandboxRetentionMode;
+    artifactRootPath?: string;
+}): Promise<TestTargetRun>;
+export declare function runTargetTestsAuto(sourcePath: string, spec: LoadedTestSpec, targets: TestTargetCandidate[], options?: {
+    executor?: WorkflowExecutor | null;
+    preserveSandboxes?: TestSandboxRetentionMode;
+    artifactRootPath?: string;
+}): Promise<TestTargetRun>;
+export declare function saveTargetTestRun(artifactRootPath: string, result: TestTargetRun): string;

package/dist/packages/testing/test-execution.js

@@ -0,0 +1,525 @@
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { buildRuntimeExecutorInfo } from "../agents/lib/executors.js";
+import { targetTestRunGitignorePath, targetTestRunsPath, targetTestSandboxGitignorePath, targetTestSandboxesPath, normalizeTestId, } from "./test-paths.js";
+import { createTestSandbox, } from "./test-sandbox.js";
+function parseWords(content) {
+    return content.trim().split(/\s+/).filter(Boolean).length;
+}
+function normalizeText(content) {
+    return content.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function evaluateTextExpect(testCase, content) {
+    const checks = [];
+    const normalized = normalizeText(content);
+    const wordCount = parseWords(content);
+    for (const phrase of testCase.expect?.must_include ?? []) {
+        const ok = normalized.includes(normalizeText(phrase));
+        checks.push({
+            label: `must include "${phrase}"`,
+            ok,
+            detail: ok ? "present" : "missing",
+        });
+    }
+    for (const options of testCase.expect?.must_include_one_of ?? []) {
+        const ok = options.some((phrase) => normalized.includes(normalizeText(phrase)));
+        checks.push({
+            label: `must include one of [${options.map((phrase) => `"${phrase}"`).join(", ")}]`,
+            ok,
+            detail: ok ? "present" : "missing",
+        });
+    }
+    for (const phrase of testCase.expect?.must_not_include ?? []) {
+        const ok = !normalized.includes(normalizeText(phrase));
+        checks.push({
+            label: `must not include "${phrase}"`,
+            ok,
+            detail: ok ? "absent" : "present",
+        });
+    }
+    if (typeof testCase.expect?.min_words === "number") {
+        const ok = wordCount >= testCase.expect.min_words;
+        checks.push({
+            label: `min words ${testCase.expect.min_words}`,
+            ok,
+            detail: `${wordCount} words`,
+        });
+    }
+    if (typeof testCase.expect?.max_words === "number") {
+        const ok = wordCount <= testCase.expect.max_words;
+        checks.push({
+            label: `max words ${testCase.expect.max_words}`,
+            ok,
+            detail: `${wordCount} words`,
+        });
+    }
+    return {
+        checks,
+        wordCount,
+    };
+}
+function testCaseNeedsExecutor(testCase) {
+    return !testCase.file || Boolean(testCase.answer);
+}
+function runTestCase(target, testCase) {
+    if (!testCase.file) {
+        throw new Error(`Test case "${testCase.id}" requires an executor because it has no file target.`);
+    }
+    const outputPath = join(target.path, testCase.file);
+    const checks = [];
+    if (!existsSync(outputPath)) {
+        checks.push({
+            label: "file exists",
+            ok: false,
+            detail: `Missing ${testCase.file}`,
+        });
+        return {
+            caseId: testCase.id,
+            question: testCase.question,
+            ...(testCase.file ? { file: testCase.file } : {}),
+            ok: false,
+            wordCount: 0,
+            passedChecks: 0,
+            totalChecks: checks.length,
+            checks,
+        };
+    }
+    checks.push({
+        label: "file exists",
+        ok: true,
+        detail: `Found ${testCase.file}`,
+    });
+    const content = readFileSync(outputPath, "utf8");
+    const evaluated = evaluateTextExpect(testCase, content);
+    const wordCount = evaluated.wordCount;
+    checks.push(...evaluated.checks);
+    const passedChecks = checks.filter((check) => check.ok).length;
+    return {
+        caseId: testCase.id,
+        question: testCase.question,
+        ...(testCase.file ? { file: testCase.file } : {}),
+        ok: passedChecks === checks.length,
+        wordCount,
+        passedChecks,
+        totalChecks: checks.length,
+        checks,
+    };
+}
+function buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath) {
+    return [
+        "You are judging whether one Interf test answer passes.",
+        "Do not browse other files or ask follow-up questions.",
+        "Judge only from the check rule and the candidate answer below.",
+        "Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
+        `Write JSON to ${JSON.stringify(verdictPath)} with keys: pass (boolean), summary (string).`,
+        "Before finishing, write the JSON verdict file.",
+        "Final line must be `DONE: pass=true - <short summary>` or `DONE: pass=false - <short summary>`.",
+        `Question: ${testCase.question}`,
+        `Expected answer: ${testCase.answer ?? "The answer clearly satisfies the question."}`,
+        `Strictness: ${testCase.strictness ?? "approximate"}`,
+        `Candidate: ${candidateLabel}`,
+        "Candidate answer starts after the next line and ends at `END CANDIDATE`.",
+        candidateContent,
+        "END CANDIDATE",
+    ].join("\n");
+}
+function readTestJudgeVerdict(verdictPath) {
+    if (!existsSync(verdictPath))
+        return null;
+    const raw = JSON.parse(readFileSync(verdictPath, "utf8"));
+    return {
+        pass: raw.pass === true,
+        summary: typeof raw.summary === "string" ? raw.summary : "",
+    };
+}
+function readTestJudgeVerdictFromStatus(statusPath) {
+    if (!existsSync(statusPath))
+        return null;
+    const lines = readFileSync(statusPath, "utf8")
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0);
+    for (let index = lines.length - 1; index >= 0; index -= 1) {
+        const line = lines[index];
+        if (!line)
+            continue;
+        if (!/^(DONE|BLOCKED|ERROR):/i.test(line))
+            continue;
+        const normalized = line.toLowerCase();
+        const summary = line.replace(/^(DONE|BLOCKED|ERROR):\s*/i, "").trim();
+        if (normalized.includes("pass=true")) {
+            return { pass: true, summary };
+        }
+        if (normalized.includes("pass=false")) {
+            return { pass: false, summary };
+        }
+        if (/values match expected|matches expected|candidate matches|answer matches|expected values match/i.test(line)) {
+            return { pass: true, summary };
+        }
+        if (/does not match|do not match|mismatch|candidate fails|answer fails|expected values do not match/i.test(line)) {
+            return { pass: false, summary };
+        }
+    }
+    return null;
+}
+async function runTargetTestsJudge(testCase, executor, candidateLabel, candidateContent) {
+    const tempDir = mkdtempSync(join(tmpdir(), "interf-test-judge-"));
+    let executionError = null;
+    let verdict = null;
+    try {
+        const verdictPath = join(tempDir, "verdict.json");
+        const statusPath = join(tempDir, "judge.status.log");
+        const prompt = buildTestJudgePrompt(testCase, candidateLabel, candidateContent, verdictPath);
+        try {
+            await executor.execute(tempDir, prompt, {
+                statusLogPath: statusPath,
+            });
+        }
+        catch (error) {
+            executionError = error instanceof Error ? error.message : String(error);
+        }
+        try {
+            verdict = readTestJudgeVerdict(verdictPath);
+            if (!verdict) {
+                verdict = readTestJudgeVerdictFromStatus(statusPath);
+            }
+        }
+        catch (error) {
+            executionError = error instanceof Error ? error.message : String(error);
+        }
+    }
+    finally {
+        rmSync(tempDir, { recursive: true, force: true });
+    }
+    return { verdict, error: executionError };
+}
+function buildTestQueryPrompt(target, testCase, answerPath, tracePath) {
+    const header = target.type === "compiled"
+        ? [
+            "You are running an Interf test inside an isolated sandboxed compiled context.",
+            "Read `AGENTS.md` first.",
+            "Use the local native `interf-query` skill available in this context folder.",
+            "Answer the check question the same way you would answer a real user inside this context folder.",
+            "Prefer the workflow-declared context outputs before raw fallback.",
+            "This sandbox is self-contained: the copied context folder has its own sanitized `raw/` fallback via `.interf/interf.json` `source.path`.",
+            "The original project root is intentionally absent from this sandbox. Work only from this sandboxed compiled context and its embedded raw files.",
+        ]
+        : [
+            "You are running an Interf baseline test inside an isolated raw test shell.",
+            "Read `AGENTS.md` first.",
+            "Use the local native `interf-query` skill available in this shell.",
+            "There is no compiled context in this sandbox.",
+            "Answer only from `raw/` inside this shell.",
+            "The original project root is intentionally absent from this sandbox.",
+        ];
+    return [
+        ...header,
+        "Emit only STATUS:, DONE:, BLOCKED:, or ERROR: lines.",
+        "Do not ask follow-up questions.",
+        `Write the answer to ${JSON.stringify(answerPath)}.`,
+        `Write the trace to ${JSON.stringify(tracePath)} with keys: case_id, target, artifacts_consulted, raw_paths_read, used_raw_fallback, answer_summary.`,
+        `Set \`case_id\` to ${JSON.stringify(testCase.id)}.`,
+        `Set \`target\` to ${JSON.stringify(target.type)}.`,
+        `Question: ${testCase.question}`,
+    ].join("\n");
+}
+async function runLiveTestCase(target, testCase, executor) {
+    const tempDir = mkdtempSync(join(tmpdir(), "interf-test-live-"));
+    const answerPath = join(tempDir, "answer.md");
+    const tracePath = join(tempDir, "trace.json");
+    const statusPath = join(tempDir, "status.log");
+    const eventPath = join(tempDir, "events.ndjson");
+    const prompt = buildTestQueryPrompt(target, testCase, answerPath, tracePath);
+    let executionError = null;
+    let code = -1;
+    try {
+        try {
+            code = await executor.execute(target.path, prompt, {
+                eventLogPath: eventPath,
+                statusLogPath: statusPath,
+            });
+        }
+        catch (error) {
+            executionError = error instanceof Error ? error.message : String(error);
+        }
+        const checks = [];
+        if (!existsSync(answerPath)) {
+            checks.push({
+                label: "answer exists",
+                ok: false,
+                detail: executionError ? `missing answer file (${executionError})` : "missing answer file",
+            });
+            return {
+                caseId: testCase.id,
+                question: testCase.question,
+                ok: false,
+                wordCount: 0,
+                passedChecks: 0,
+                totalChecks: checks.length,
+                checks,
+            };
+        }
+        const answer = readFileSync(answerPath, "utf8");
+        const evaluated = evaluateTextExpect(testCase, answer);
+        checks.push({
+            label: "answer exists",
+            ok: true,
+            detail: "present",
+        });
+        checks.push(...evaluated.checks);
+        let trace = null;
+        if (existsSync(tracePath)) {
+            try {
+                trace = JSON.parse(readFileSync(tracePath, "utf8"));
+                checks.push({
+                    label: "trace exists",
+                    ok: true,
+                    detail: "present",
+                });
+            }
+            catch (error) {
+                checks.push({
+                    label: "trace exists",
+                    ok: false,
+                    detail: `invalid JSON: ${error instanceof Error ? error.message : String(error)}`,
+                });
+            }
+        }
+        else {
+            checks.push({
+                label: "trace exists",
+                ok: false,
+                detail: "missing trace file",
+            });
+        }
+        if (testCase.answer) {
+            const judged = await runTargetTestsJudge(testCase, executor, `generated answer for ${testCase.id}`, answer);
+            checks.push({
+                label: "judge verdict",
+                ok: judged.verdict?.pass === true,
+                detail: judged.verdict?.summary && judged.verdict.summary.trim().length > 0
+                    ? judged.verdict.summary.trim()
+                    : judged.error ?? "missing or invalid judge verdict",
+            });
+        }
+        const passedChecks = checks.filter((check) => check.ok).length;
+        return {
+            caseId: testCase.id,
+            question: testCase.question,
+            ok: code === 0 && passedChecks === checks.length,
+            wordCount: evaluated.wordCount,
+            passedChecks,
+            totalChecks: checks.length,
+            checks,
+            answer,
+            ...(trace ? { trace } : {}),
+        };
+    }
+    finally {
+        rmSync(tempDir, { recursive: true, force: true });
+    }
+}
+async function runTestCaseWithJudge(target, testCase, executor) {
+    if (!testCase.file) {
+        return runLiveTestCase(target, testCase, executor);
+    }
+    const outputPath = join(target.path, testCase.file);
+    const checks = [];
+    if (!existsSync(outputPath)) {
+        checks.push({
+            label: "file exists",
+            ok: false,
+            detail: `Missing ${testCase.file}`,
+        });
+        return {
+            caseId: testCase.id,
+            question: testCase.question,
+            ...(testCase.file ? { file: testCase.file } : {}),
+            ok: false,
+            wordCount: 0,
+            passedChecks: 0,
+            totalChecks: checks.length,
+            checks,
+        };
+    }
+    const content = readFileSync(outputPath, "utf8");
+    const evaluated = evaluateTextExpect(testCase, content);
+    const wordCount = evaluated.wordCount;
+    checks.push({
+        label: "file exists",
+        ok: true,
+        detail: `Found ${testCase.file}`,
+    });
+    checks.push(...evaluated.checks);
+    if (testCase.answer) {
+        const judged = await runTargetTestsJudge(testCase, executor, `compiled file ${outputPath}`, content);
+        checks.push({
+            label: "judge verdict",
+            ok: judged.verdict?.pass === true,
+            detail: judged.verdict?.summary && judged.verdict.summary.trim().length > 0
+                ? judged.verdict.summary.trim()
+                : judged.error ?? "missing or invalid judge verdict",
+        });
+    }
+    const passedChecks = checks.filter((check) => check.ok).length;
+    return {
+        caseId: testCase.id,
+        question: testCase.question,
+        ...(testCase.file ? { file: testCase.file } : {}),
+        ok: passedChecks === checks.length,
+        wordCount,
+        passedChecks,
+        totalChecks: checks.length,
+        checks,
+    };
+}
+function buildTestTargetResult(target, caseResults, options = {}) {
+    const passedCases = caseResults.filter((result) => result.ok).length;
+    const passedChecks = caseResults.reduce((total, result) => total + result.passedChecks, 0);
+    const totalChecks = caseResults.reduce((total, result) => total + result.totalChecks, 0);
+    return {
+        target: {
+            type: target.type,
+            name: target.name,
+            path: target.path,
+            workflow: target.workflow,
+        },
+        ...(options.sandboxPath ? { sandbox_path: options.sandboxPath } : {}),
+        ok: passedCases === caseResults.length,
+        passedCases,
+        totalCases: caseResults.length,
+        passedChecks,
+        totalChecks,
+        caseResults,
+    };
+}
+function buildTestTargetRun(sourcePath, spec, results, executor, generatedAt) {
+    return {
+        kind: "interf-test-target-run",
+        version: 1,
+        generated_at: generatedAt ?? new Date().toISOString(),
+        spec: {
+            id: spec.id,
+            name: spec.name,
+            type: spec.type,
+            file: spec.filePath,
+            ...(spec.description ? { description: spec.description } : {}),
+            case_count: spec.cases.length,
+            cases: spec.cases.map((testCase) => ({
+                id: testCase.id,
+                question: testCase.question,
+                ...(testCase.file ? { file: testCase.file } : {}),
+                ...(testCase.answer ? { answer: testCase.answer } : {}),
+                ...(testCase.strictness ? { strictness: testCase.strictness } : {}),
+                ...(testCase.expect ? { expect: testCase.expect } : {}),
+            })),
+        },
+        source_path: sourcePath,
+        ...(executor ? { executor: buildRuntimeExecutorInfo(executor) } : {}),
+        target_count: results.length,
+        results,
+    };
+}
+export function runTargetTests(sourcePath, spec, targets) {
+    if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
+        throw new Error("This test needs a live executor. Use runTargetTestsWithJudge instead.");
+    }
+    for (const target of targets) {
+        if (target.type !== spec.type) {
+            throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
+        }
+    }
+    const results = targets.map((target) => buildTestTargetResult(target, spec.cases.map((testCase) => runTestCase(target, testCase))));
+    return buildTestTargetRun(sourcePath, spec, results);
+}
+export async function runTargetTestsWithJudge(sourcePath, spec, targets, executor, options = {}) {
+    const preserveMode = options.preserveSandboxes ?? "on-failure";
+    const artifactRootPath = options.artifactRootPath ?? sourcePath;
+    const generatedAt = new Date().toISOString();
+    const sandboxRunId = `${generatedAt.replace(/[:.]/g, "-")}-${spec.id}`;
+    for (const target of targets) {
+        if (target.type !== spec.type) {
+            throw new Error(`Test target type mismatch: expected ${spec.type}, got ${target.type}`);
+        }
+    }
+    const results = [];
+    for (const [index, target] of targets.entries()) {
+        const sandbox = createTestSandbox(target);
+        try {
+            const sandboxTarget = {
+                ...target,
+                path: sandbox.targetPath,
+            };
+            const caseResults = [];
+            for (const testCase of spec.cases) {
+                if (testCaseNeedsExecutor(testCase)) {
+                    caseResults.push(await runLiveTestCase(sandboxTarget, testCase, executor));
+                }
+                else {
+                    caseResults.push(await runTestCaseWithJudge(sandboxTarget, testCase, executor));
+                }
+            }
+            let sandboxPath;
+            const targetResult = buildTestTargetResult(target, caseResults);
+            const shouldPreserveSandbox = preserveMode === "always" || !targetResult.ok;
+            if (shouldPreserveSandbox) {
+                const sandboxRoot = targetTestSandboxesPath(artifactRootPath, target.type);
+                mkdirSync(sandboxRoot, { recursive: true });
+                const gitignorePath = targetTestSandboxGitignorePath(artifactRootPath, target.type);
+                if (!existsSync(gitignorePath)) {
+                    writeFileSync(gitignorePath, "*\n!.gitignore\n");
+                }
+                const sandboxPathName = `${String(index + 1).padStart(2, "0")}-${normalizeTestId(target.name) || target.type}`;
+                sandbox.preserve(join(sandboxRoot, sandboxRunId, sandboxPathName));
+                sandboxPath = sandbox.targetPath;
+            }
+            results.push(buildTestTargetResult(target, caseResults, { sandboxPath }));
+        }
+        finally {
+            sandbox.cleanup();
+        }
+    }
+    return buildTestTargetRun(sourcePath, spec, results, executor, generatedAt);
+}
+export async function runTargetTestsAuto(sourcePath, spec, targets, options) {
+    if (spec.cases.some((testCase) => testCaseNeedsExecutor(testCase))) {
+        if (!options?.executor) {
+            throw new Error("This test needs a live local executor, but no executor was provided.");
+        }
+        return runTargetTestsWithJudge(sourcePath, spec, targets, options.executor, {
+            preserveSandboxes: options.preserveSandboxes,
+            artifactRootPath: options.artifactRootPath,
+        });
+    }
+    return runTargetTests(sourcePath, spec, targets);
+}
+export function saveTargetTestRun(artifactRootPath, result) {
+    const dirPath = targetTestRunsPath(artifactRootPath, result.spec.type);
+    mkdirSync(dirPath, { recursive: true });
+    const gitignorePath = targetTestRunGitignorePath(artifactRootPath, result.spec.type);
+    if (!existsSync(gitignorePath)) {
+        writeFileSync(gitignorePath, "*\n!.gitignore\n");
+    }
+    const timestamp = result.generated_at.replace(/[:.]/g, "-");
+    const runDirPath = join(dirPath, `${timestamp}-${result.spec.id}`);
+    mkdirSync(runDirPath, { recursive: true });
+    const manifestPath = join(runDirPath, "manifest.json");
+    writeFileSync(manifestPath, `${JSON.stringify({
+        kind: "interf-test-target-run-manifest",
+        version: 1,
+        generated_at: result.generated_at,
+        spec: {
+            id: result.spec.id,
+            name: result.spec.name,
+            type: result.spec.type,
+            case_count: result.spec.case_count,
+        },
+        result_file: "run.json",
+        target_count: result.target_count,
+        passed_targets: result.results.filter((entry) => entry.ok).length,
+    }, null, 2)}\n`);
+    const runPath = join(runDirPath, "run.json");
+    writeFileSync(runPath, `${JSON.stringify(result, null, 2)}\n`);
+    return runPath;
+}