@interf/compiler 0.22.2 → 0.33.0

package/dist/packages/runtime/verify/benchmark-run.js

@@ -0,0 +1,303 @@
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { buildTestSpecFromProjectConfig, buildTestSpecFromSourceFolderConfig, fingerprintSavedBenchmarkSpec, } from "../../projects/source-config.js";
+import { asProjectDataDir, projectLatestBenchmarkRunPath, projectLatestBenchmarkSummaryPath, projectTestRunPath, projectTestRunsRoot, projectTestsRoot, normalizeProjectTestRunId, projectContextGraphPath, } from "../../contracts/lib/project-paths.js";
+import { testRootForContextGraph } from "../build/context-graph-paths.js";
+import { readJsonFileWithSchema } from "../../contracts/utils/parse.js";
+import { resolveBuildPlanId } from "../../build-plans/build-plan-resolution.js";
+import { createContextGraphTestTarget, createSourceFilesTestTarget, runTargetTestsAuto, saveTargetTestRun, } from "./verify.js";
+import { BenchmarkRunSchema, } from "./lib/schema.js";
+export function benchmarkPassRate(outcome) {
+    return outcome.result.totalCases > 0
+        ? Math.round((outcome.result.passedCases / outcome.result.totalCases) * 100)
+        : 0;
+}
+function visibleRunPath(outcome) {
+    return outcome.displayRunPath ?? outcome.runPath;
+}
+function traceUsesSourceFiles(trace) {
+    if (!trace || typeof trace !== "object" || Array.isArray(trace))
+        return false;
+    const record = trace;
+    if (record.used_source_files === true)
+        return true;
+    const sourcePaths = record.source_paths_read;
+    return Array.isArray(sourcePaths) && sourcePaths.some((entry) => typeof entry === "string" && entry.trim().length > 0);
+}
+function sourceUsageForOutcome(outcome) {
+    if (outcome.target.type === "source-files")
+        return "source-baseline";
+    const caseResults = outcome.result.caseResults ?? [];
+    if (caseResults.length === 0)
+        return "unknown";
+    return caseResults.some((result) => traceUsesSourceFiles(result.trace))
+        ? "graph-source-verified"
+        : "graph-only";
+}
+function summarizeSavedTestOutcome(label, outcome) {
+    return {
+        label,
+        run_path: visibleRunPath(outcome),
+        ok: outcome.result.ok,
+        passed_cases: outcome.result.passedCases,
+        total_cases: outcome.result.totalCases,
+        passed_checks: outcome.result.passedChecks,
+        total_checks: outcome.result.totalChecks,
+        executor: outcome.executor ?? null,
+        source_usage: sourceUsageForOutcome(outcome),
+        target: outcome.target,
+    };
+}
+function writeProjectTargetRun(options) {
+    const dirPath = projectTestRunsRoot(asProjectDataDir(options.projectDataDir), options.projectName, options.target);
+    mkdirSync(dirPath, { recursive: true });
+    const runPath = projectTestRunPath(asProjectDataDir(options.projectDataDir), options.projectName, options.target, options.generatedAt, options.runId, options.runSuffix);
+    writeFileSync(runPath, `${JSON.stringify(options.payload, null, 2)}\n`);
+    return runPath;
+}
+function renderLatestSummaryMarkdown(payload) {
+    const lines = [
+        "# Latest Benchmark Results",
+        "",
+        "| Target | Benchmark checks | Source use |",
+        "| --- | --- | --- |",
+    ];
+    if (payload.source_files) {
+        lines.push(`| Source files | \`${payload.source_files.passed_cases}/${payload.source_files.total_cases}\` | ${payload.source_files.source_usage} |`);
+    }
+    if (payload.context_graph) {
+        lines.push(`| Context Graph | \`${payload.context_graph.passed_cases}/${payload.context_graph.total_cases}\` | ${payload.context_graph.source_usage} |`);
+    }
+    lines.push("");
+    if (payload.source_files) {
+        lines.push(`- Latest source-files run: ${payload.source_files.run_path}`);
+    }
+    if (payload.context_graph) {
+        lines.push(`- Latest context-graph run: ${payload.context_graph.run_path}`);
+    }
+    return `${lines.join("\n")}\n`;
+}
+export function readSavedBenchmarkRun(projectDataDir, projectName) {
+    const latestPath = projectLatestBenchmarkRunPath(asProjectDataDir(projectDataDir), projectName);
+    if (!existsSync(latestPath))
+        return null;
+    return readBenchmarkRunAtPath(latestPath, "latest benchmark/evaluation run");
+}
+export function parseBenchmarkRun(value) {
+    const parsed = BenchmarkRunSchema.safeParse(value);
+    return parsed.success ? parsed.data : null;
+}
+export function readBenchmarkRunAtPath(filePath, label = "benchmark/evaluation run") {
+    return readJsonFileWithSchema(filePath, label, BenchmarkRunSchema);
+}
+export function readCurrentSavedBenchmarkRun(options) {
+    const latestRun = readSavedBenchmarkRun(options.projectDataDir, options.projectName);
+    if (!latestRun) {
+        return { benchmarkRun: null, stale: false };
+    }
+    const currentFingerprint = fingerprintSavedBenchmarkSpec({
+        projectDataDir: options.projectDataDir,
+        projectName: options.projectName,
+        targetType: latestRun.mode === "source-files" ? "source-files" : "context-graph",
+    });
+    if (!currentFingerprint) {
+        return { benchmarkRun: null, stale: true };
+    }
+    if (!latestRun.checks_fingerprint || latestRun.checks_fingerprint !== currentFingerprint) {
+        return { benchmarkRun: null, stale: true };
+    }
+    return { benchmarkRun: latestRun, stale: false };
+}
+function modeForOutcomes(sourceFilesOutcome, contextGraphOutcome) {
+    if (sourceFilesOutcome && contextGraphOutcome)
+        return "both";
+    if (sourceFilesOutcome)
+        return "source-files";
+    return "context-graph";
+}
+function passRateOf(summary) {
+    if (!summary)
+        return null;
+    if (summary.total_cases <= 0)
+        return 0;
+    return Math.round((summary.passed_cases / summary.total_cases) * 100);
+}
+export function saveBenchmarkRun(options) {
+    const generatedAt = new Date().toISOString();
+    const sourceFilesOutcome = options.sourceFilesOutcome ?? null;
+    const sourceFilesSummary = sourceFilesOutcome
+        ? summarizeSavedTestOutcome("Source Files", sourceFilesOutcome)
+        : null;
+    const contextGraphSummary = options.contextGraphOutcome
+        ? summarizeSavedTestOutcome("Context Graph", options.contextGraphOutcome)
+        : null;
+    const payload = {
+        kind: "interf-benchmark-run",
+        version: 1,
+        generated_at: generatedAt,
+        // verify carries the target it ran against on the wire so
+        // Project activity and the UI can show source-files baselines alongside
+        // context-graph judgments. `both` is reserved for future combined runs.
+        mode: modeForOutcomes(sourceFilesOutcome, options.contextGraphOutcome),
+        source_path: options.sourcePath,
+        checks_fingerprint: options.checksFingerprint,
+        project: {
+            name: options.projectName,
+            context_graph_path: options.contextGraphPath,
+        },
+        source_files: sourceFilesSummary,
+        context_graph: contextGraphSummary,
+        summary: {
+            source_files_pass_rate: passRateOf(sourceFilesSummary),
+            context_graph_pass_rate: passRateOf(contextGraphSummary),
+        },
+    };
+    const latestStatePath = projectLatestBenchmarkRunPath(asProjectDataDir(options.sourcePath), options.projectName);
+    mkdirSync(dirname(latestStatePath), { recursive: true });
+    writeFileSync(latestStatePath, `${JSON.stringify(payload, null, 2)}\n`);
+    writeFileSync(projectLatestBenchmarkSummaryPath(asProjectDataDir(options.sourcePath), options.projectName), renderLatestSummaryMarkdown(payload));
+    if (options.contextGraphPath) {
+        mkdirSync(testRootForContextGraph(options.contextGraphPath), { recursive: true });
+        writeFileSync(join(testRootForContextGraph(options.contextGraphPath), "latest.json"), `${JSON.stringify(payload, null, 2)}\n`);
+    }
+    return latestStatePath;
+}
+export async function runSavedContextGraphBenchmark(options) {
+    const contextGraphPath = options.contextGraphPath
+        ?? contextGraphPathForProject(options.sourcePath, options.projectConfig.name);
+    const spec = buildTestSpecFromProjectConfig({
+        contextGraphPath: contextGraphPath,
+        targetType: "context-graph",
+    });
+    if (!spec)
+        return null;
+    const target = createContextGraphTestTarget(contextGraphPath, options.projectConfig.name, resolveBuildPlanId(options.projectConfig));
+    if (!target.eligible)
+        return null;
+    const run = await runTargetTestsAuto(options.sourcePath, spec, [target], {
+        executor: options.executor,
+        preserveSandboxes: options.preserveSandboxes ?? "on-failure",
+        artifactRootPath: contextGraphPath,
+    });
+    const result = run.results[0];
+    if (!result)
+        return null;
+    const contextGraphRunPath = saveTargetTestRun(contextGraphPath, run);
+    const projectRunPath = writeProjectTargetRun({
+        projectDataDir: options.sourcePath,
+        projectName: options.projectConfig.name,
+        target: "context-graph",
+        generatedAt: run.generated_at,
+        runId: normalizeProjectTestRunId(spec.id),
+        runSuffix: options.runSuffix,
+        payload: run,
+    });
+    return {
+        runPath: contextGraphRunPath,
+        displayRunPath: projectRunPath,
+        executor: run.executor ?? null,
+        target,
+        result,
+    };
+}
+/**
+ * Run the saved benchmark checks against Source references from the recorded
+ * Source Manifest so users can compare raw Source access with the Context Graph.
+ */
+export async function runSavedSourceFilesBenchmark(options) {
+    const spec = buildTestSpecFromSourceFolderConfig({
+        projectDataDir: options.sourcePath,
+        targetName: options.projectConfig.name,
+        targetType: "source-files",
+    });
+    if (!spec)
+        return null;
+    if (!options.contextGraphPath)
+        return null;
+    const target = createSourceFilesTestTarget(options.contextGraphPath);
+    if (!target.eligible)
+        return null;
+    const run = await runTargetTestsAuto(options.contextGraphPath, spec, [target], {
+        executor: options.executor,
+        preserveSandboxes: options.preserveSandboxes ?? "on-failure",
+        artifactRootPath: projectTestsRoot(asProjectDataDir(options.sourcePath), options.projectConfig.name),
+    });
+    const result = run.results[0];
+    if (!result)
+        return null;
+    const projectRunPath = writeProjectTargetRun({
+        projectDataDir: options.sourcePath,
+        projectName: options.projectConfig.name,
+        target: "source-files",
+        generatedAt: run.generated_at,
+        runId: normalizeProjectTestRunId(spec.id),
+        runSuffix: options.runSuffix,
+        payload: run,
+    });
+    return {
+        runPath: projectRunPath,
+        executor: run.executor ?? null,
+        target,
+        result,
+    };
+}
+export async function runBenchmarkForExecutor(options) {
+    const target = options.target ?? "context-graph";
+    const contextGraphOutcomePromise = target === "context-graph" || target === "both"
+        ? runSavedContextGraphBenchmark({
+            sourcePath: options.sourcePath,
+            projectConfig: options.projectConfig,
+            executor: options.executor,
+            contextGraphPath: options.contextGraphPath,
+            preserveSandboxes: options.preserveSandboxes,
+            runSuffix: options.runSuffix,
+        })
+        : Promise.resolve(null);
+    const sourceFilesOutcomePromise = target === "source-files" || target === "both"
+        ? runSavedSourceFilesBenchmark({
+            sourcePath: options.sourcePath,
+            projectConfig: options.projectConfig,
+            contextGraphPath: options.contextGraphPath,
+            executor: options.executor,
+            preserveSandboxes: options.preserveSandboxes,
+            runSuffix: options.runSuffix,
+        })
+        : Promise.resolve(null);
+    const [contextGraphOutcome, sourceFilesOutcome] = await Promise.all([
+        contextGraphOutcomePromise,
+        sourceFilesOutcomePromise,
+    ]);
+    const modeSatisfied = target === "both"
+        ? Boolean(contextGraphOutcome && sourceFilesOutcome)
+        : Boolean(contextGraphOutcome || sourceFilesOutcome);
+    const shouldSave = options.saveLatest !== false;
+    const benchmarkRunPath = shouldSave && modeSatisfied
+        ? saveBenchmarkRun({
+            sourcePath: options.sourcePath,
+            contextGraphPath: options.contextGraphPath,
+            projectName: options.projectConfig.name,
+            checksFingerprint: fingerprintSavedBenchmarkSpec({
+                projectDataDir: options.sourcePath,
+                projectName: options.projectConfig.name,
+                targetType: target === "source-files" ? "source-files" : "context-graph",
+            }) ?? "missing-benchmark-spec",
+            sourceFilesOutcome,
+            contextGraphOutcome,
+        })
+        : null;
+    const benchmarkRun = shouldSave && modeSatisfied
+        ? benchmarkRunPath ? readBenchmarkRunAtPath(benchmarkRunPath) : null
+        : null;
+    return {
+        sourcePath: options.sourcePath,
+        projectConfig: options.projectConfig,
+        contextGraphPath: options.contextGraphPath,
+        sourceFilesOutcome,
+        contextGraphOutcome,
+        benchmarkRunPath,
+        benchmarkRun,
+    };
+}
+function contextGraphPathForProject(sourcePath, projectName) {
+    return projectContextGraphPath(asProjectDataDir(sourcePath), projectName);
+}

package/dist/packages/{engine → runtime}/verify/index.d.ts

@@ -3,11 +3,11 @@ export * as verify from "./verify.js";
 export * as execution from "./verify-execution.js";
 export * as paths from "./verify-paths.js";
 export * as profilePresets from "./verify-profile-presets.js";
-export * as readinessCheckRun from "./readiness-check-run.js";
+export * as benchmarkRun from "./benchmark-run.js";
 export * as sandbox from "./verify-sandbox.js";
 export * as specs from "./verify-specs.js";
 export * as targets from "./verify-targets.js";
 export * as types from "./verify-types.js";
-export type { TestCase, TestCaseExpect, TestCheckResult, TestCaseResult, TestSpec, TestTargetResult, TestTargetRun, ReadinessCheckRun, TestTargetType, } from "./lib/schema.js";
+export type { TestCase, TestCaseExpect, TestCheckResult, TestCaseResult, TestSpec, TestTargetResult, TestTargetRun, BenchmarkRun, TestTargetType, } from "./lib/schema.js";
 export type { LoadedTestSpec, TestTargetCandidate, } from "./verify-types.js";
 export type { TestSandboxRetentionMode, } from "./verify-sandbox.js";

package/dist/packages/{engine → runtime}/verify/index.js

@@ -3,7 +3,7 @@ export * as verify from "./verify.js";
 export * as execution from "./verify-execution.js";
 export * as paths from "./verify-paths.js";
 export * as profilePresets from "./verify-profile-presets.js";
-export * as readinessCheckRun from "./readiness-check-run.js";
+export * as benchmarkRun from "./benchmark-run.js";
 export * as sandbox from "./verify-sandbox.js";
 export * as specs from "./verify-specs.js";
 export * as targets from "./verify-targets.js";

package/dist/packages/{engine → runtime}/verify/lib/schema.d.ts

@@ -16,8 +16,8 @@ export declare const TestCaseSchema: z.ZodObject<{
 }, z.core.$strip>;
 export declare const TestSpecSchema: z.ZodObject<{
     type: z.ZodEnum<{
-        "verifiable-context": "verifiable-context";
         "source-files": "source-files";
+        "context-graph": "context-graph";
     }>;
     name: z.ZodString;
     description: z.ZodOptional<z.ZodString>;
@@ -60,8 +60,8 @@ export declare const TestCaseResultSchema: z.ZodObject<{
 export declare const TestTargetResultSchema: z.ZodObject<{
     target: z.ZodObject<{
         type: z.ZodEnum<{
-            "verifiable-context": "verifiable-context";
             "source-files": "source-files";
+            "context-graph": "context-graph";
         }>;
         name: z.ZodString;
         path: z.ZodString;
@@ -98,8 +98,8 @@ export declare const TestTargetRunSchema: z.ZodObject<{
         id: z.ZodString;
         name: z.ZodString;
         type: z.ZodEnum<{
-            "verifiable-context": "verifiable-context";
             "source-files": "source-files";
+            "context-graph": "context-graph";
         }>;
         file: z.ZodString;
         description: z.ZodOptional<z.ZodString>;
@@ -138,8 +138,8 @@ export declare const TestTargetRunSchema: z.ZodObject<{
     results: z.ZodArray<z.ZodObject<{
         target: z.ZodObject<{
             type: z.ZodEnum<{
-                "verifiable-context": "verifiable-context";
                 "source-files": "source-files";
+                "context-graph": "context-graph";
             }>;
             name: z.ZodString;
             path: z.ZodString;
@@ -170,10 +170,16 @@ export declare const TestTargetRunSchema: z.ZodObject<{
     }, z.core.$strip>>;
 }, z.core.$strip>;
 export declare const TestRunModeSchema: z.ZodEnum<{
-    "verifiable-context": "verifiable-context";
     "source-files": "source-files";
+    "context-graph": "context-graph";
     both: "both";
 }>;
+export declare const BenchmarkSourceUsageSchema: z.ZodEnum<{
+    unknown: "unknown";
+    "source-baseline": "source-baseline";
+    "graph-only": "graph-only";
+    "graph-source-verified": "graph-source-verified";
+}>;
 export declare const TestRunTargetSummarySchema: z.ZodObject<{
     label: z.ZodString;
     run_path: z.ZodString;
@@ -182,30 +188,50 @@ export declare const TestRunTargetSummarySchema: z.ZodObject<{
     total_cases: z.ZodNumber;
     passed_checks: z.ZodNumber;
     total_checks: z.ZodNumber;
+    executor: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "local-agent": "local-agent";
+            "connected-provider": "connected-provider";
+            managed: "managed";
+        }>;
+        name: z.ZodString;
+        display_name: z.ZodString;
+        command: z.ZodNullable<z.ZodString>;
+        model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        effort: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        profile: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        timeout_ms: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strip>>>;
+    source_usage: z.ZodDefault<z.ZodEnum<{
+        unknown: "unknown";
+        "source-baseline": "source-baseline";
+        "graph-only": "graph-only";
+        "graph-source-verified": "graph-source-verified";
+    }>>;
     target: z.ZodObject<{
         type: z.ZodEnum<{
-            "verifiable-context": "verifiable-context";
             "source-files": "source-files";
+            "context-graph": "context-graph";
         }>;
         name: z.ZodString;
         path: z.ZodString;
         build_plan: z.ZodString;
     }, z.core.$strip>;
 }, z.core.$strip>;
-export declare const ReadinessCheckRunSchema: z.ZodPreprocess<z.ZodObject<{
-    kind: z.ZodLiteral<"interf-readiness-check-run">;
+export declare const BenchmarkRunSchema: z.ZodPipe<z.ZodTransform<unknown, unknown>, z.ZodObject<{
+    kind: z.ZodLiteral<"interf-benchmark-run">;
     version: z.ZodLiteral<1>;
     generated_at: z.ZodString;
     mode: z.ZodEnum<{
-        "verifiable-context": "verifiable-context";
         "source-files": "source-files";
+        "context-graph": "context-graph";
         both: "both";
     }>;
     source_path: z.ZodString;
     checks_fingerprint: z.ZodOptional<z.ZodString>;
-    preparation: z.ZodObject<{
+    project: z.ZodObject<{
         name: z.ZodString;
-        verifiable_context_path: z.ZodNullable<z.ZodString>;
+        context_graph_path: z.ZodNullable<z.ZodString>;
     }, z.core.$strip>;
     source_files: z.ZodNullable<z.ZodObject<{
         label: z.ZodString;
@@ -215,17 +241,37 @@ export declare const ReadinessCheckRunSchema: z.ZodPreprocess<z.ZodObject<{
         total_cases: z.ZodNumber;
         passed_checks: z.ZodNumber;
         total_checks: z.ZodNumber;
+        executor: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+            kind: z.ZodEnum<{
+                "local-agent": "local-agent";
+                "connected-provider": "connected-provider";
+                managed: "managed";
+            }>;
+            name: z.ZodString;
+            display_name: z.ZodString;
+            command: z.ZodNullable<z.ZodString>;
+            model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            effort: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            profile: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            timeout_ms: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+        }, z.core.$strip>>>;
+        source_usage: z.ZodDefault<z.ZodEnum<{
+            unknown: "unknown";
+            "source-baseline": "source-baseline";
+            "graph-only": "graph-only";
+            "graph-source-verified": "graph-source-verified";
+        }>>;
         target: z.ZodObject<{
             type: z.ZodEnum<{
-                "verifiable-context": "verifiable-context";
                 "source-files": "source-files";
+                "context-graph": "context-graph";
             }>;
             name: z.ZodString;
             path: z.ZodString;
             build_plan: z.ZodString;
         }, z.core.$strip>;
     }, z.core.$strip>>;
-    verifiable_context: z.ZodNullable<z.ZodObject<{
+    context_graph: z.ZodNullable<z.ZodObject<{
         label: z.ZodString;
         run_path: z.ZodString;
         ok: z.ZodBoolean;
@@ -233,10 +279,30 @@ export declare const ReadinessCheckRunSchema: z.ZodPreprocess<z.ZodObject<{
         total_cases: z.ZodNumber;
         passed_checks: z.ZodNumber;
         total_checks: z.ZodNumber;
+        executor: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+            kind: z.ZodEnum<{
+                "local-agent": "local-agent";
+                "connected-provider": "connected-provider";
+                managed: "managed";
+            }>;
+            name: z.ZodString;
+            display_name: z.ZodString;
+            command: z.ZodNullable<z.ZodString>;
+            model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            effort: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            profile: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            timeout_ms: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+        }, z.core.$strip>>>;
+        source_usage: z.ZodDefault<z.ZodEnum<{
+            unknown: "unknown";
+            "source-baseline": "source-baseline";
+            "graph-only": "graph-only";
+            "graph-source-verified": "graph-source-verified";
+        }>>;
         target: z.ZodObject<{
             type: z.ZodEnum<{
-                "verifiable-context": "verifiable-context";
                 "source-files": "source-files";
+                "context-graph": "context-graph";
             }>;
             name: z.ZodString;
             path: z.ZodString;
@@ -245,7 +311,7 @@ export declare const ReadinessCheckRunSchema: z.ZodPreprocess<z.ZodObject<{
     }, z.core.$strip>>;
     summary: z.ZodObject<{
         source_files_pass_rate: z.ZodNullable<z.ZodNumber>;
-        verifiable_context_pass_rate: z.ZodNullable<z.ZodNumber>;
+        context_graph_pass_rate: z.ZodNullable<z.ZodNumber>;
     }, z.core.$strip>;
 }, z.core.$strip>>;
 export type { TestCaseExpect, TestTargetType };
@@ -256,5 +322,6 @@ export type TestCaseResult = z.infer<typeof TestCaseResultSchema>;
 export type TestTargetResult = z.infer<typeof TestTargetResultSchema>;
 export type TestTargetRun = z.infer<typeof TestTargetRunSchema>;
 export type TestRunMode = z.infer<typeof TestRunModeSchema>;
+export type BenchmarkSourceUsage = z.infer<typeof BenchmarkSourceUsageSchema>;
 export type TestRunTargetSummary = z.infer<typeof TestRunTargetSummarySchema>;
-export type ReadinessCheckRun = z.infer<typeof ReadinessCheckRunSchema>;
+export type BenchmarkRun = z.infer<typeof BenchmarkRunSchema>;

package/dist/packages/{engine → runtime}/verify/lib/schema.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, PreparationNameSchema, } from "../../../contracts/lib/schema.js";
+import { RuntimeExecutorInfoSchema, TestCaseExpectSchema, TestTargetTypeSchema, ProjectIdSchema, } from "../../../contracts/lib/schema.js";
 const TestCaseCoreSchema = z.object({
     id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,79}$/),
     question: z.string().min(1),
@@ -81,7 +81,13 @@ export const TestTargetRunSchema = z.object({
     target_count: z.number(),
     results: z.array(TestTargetResultSchema),
 });
-export const TestRunModeSchema = z.enum(["source-files", "verifiable-context", "both"]);
+export const TestRunModeSchema = z.enum(["source-files", "context-graph", "both"]);
+export const BenchmarkSourceUsageSchema = z.enum([
+    "source-baseline",
+    "graph-only",
+    "graph-source-verified",
+    "unknown",
+]);
 export const TestRunTargetSummarySchema = z.object({
     label: z.string(),
     run_path: z.string(),
@@ -90,6 +96,8 @@ export const TestRunTargetSummarySchema = z.object({
     total_cases: z.number(),
     passed_checks: z.number(),
     total_checks: z.number(),
+    executor: RuntimeExecutorInfoSchema.nullable().optional(),
+    source_usage: BenchmarkSourceUsageSchema.default("unknown"),
     target: z.object({
         type: TestTargetTypeSchema,
         name: z.string(),
@@ -97,12 +105,12 @@ export const TestRunTargetSummarySchema = z.object({
         build_plan: z.string(),
     }),
 });
-function mapReadinessCheckRun(value) {
+function mapBenchmarkRun(value) {
     if (!value || typeof value !== "object" || Array.isArray(value))
         return value;
     const record = value;
-    const preparation = record.preparation && typeof record.preparation === "object" && !Array.isArray(record.preparation)
-        ? record.preparation
+    const project = record.project && typeof record.project === "object" && !Array.isArray(record.project)
+        ? record.project
         : null;
     const summary = record.summary && typeof record.summary === "object" && !Array.isArray(record.summary)
         ? record.summary
@@ -110,36 +118,36 @@ function mapReadinessCheckRun(value) {
     return {
         ...record,
         mode: record.mode,
-        preparation: preparation
+        project: project
             ? {
-                ...preparation,
-                verifiable_context_path: preparation.verifiable_context_path,
+                ...project,
+                context_graph_path: project.context_graph_path,
             }
-            : record.preparation,
-        verifiable_context: record.verifiable_context,
+            : record.project,
+        context_graph: record.context_graph,
         summary: summary
             ? {
                 ...summary,
-                verifiable_context_pass_rate: summary.verifiable_context_pass_rate,
+                context_graph_pass_rate: summary.context_graph_pass_rate,
             }
             : record.summary,
     };
 }
-export const ReadinessCheckRunSchema = z.preprocess(mapReadinessCheckRun, z.object({
-    kind: z.literal("interf-readiness-check-run"),
+export const BenchmarkRunSchema = z.preprocess(mapBenchmarkRun, z.object({
+    kind: z.literal("interf-benchmark-run"),
     version: z.literal(1),
     generated_at: z.string(),
     mode: TestRunModeSchema,
     source_path: z.string(),
     checks_fingerprint: z.string().min(1).optional(),
-    preparation: z.object({
-        name: PreparationNameSchema,
-        verifiable_context_path: z.string().nullable(),
+    project: z.object({
+        name: ProjectIdSchema,
+        context_graph_path: z.string().nullable(),
     }),
     source_files: TestRunTargetSummarySchema.nullable(),
-    verifiable_context: TestRunTargetSummarySchema.nullable(),
+    context_graph: TestRunTargetSummarySchema.nullable(),
     summary: z.object({
         source_files_pass_rate: z.number().nullable(),
-        verifiable_context_pass_rate: z.number().nullable(),
+        context_graph_pass_rate: z.number().nullable(),
     }),
 }));

package/dist/packages/{engine → runtime}/verify/verify-execution.js

@@ -195,22 +195,22 @@ async function runTargetTestsJudge(testCase, executor, candidateLabel, candidate
     return { verdict, error: executionError };
 }
 function buildTestQueryPrompt(target, testCase, answerPath, tracePath) {
-    const header = target.type === "verifiable-context"
+    const header = target.type === "context-graph"
         ? [
-            "You are running an Interf test inside an isolated sandboxed verifiable-context copy.",
+            "You are running an Interf benchmark inside an isolated sandboxed context-graph copy.",
             "Read `AGENTS.md` first.",
-            "Use the local native `interf-query` skill available in this verifiable context.",
-            "Answer the check question the same way you would answer a real user inside this verifiable context.",
-            "Prefer the Build Plan-declared verifiable-context outputs before consulting source references.",
-            "This sandbox is self-contained: the copied verifiable context has source references in `.interf/runtime/source-snapshot.json`.",
-            "The original Interf instance state is intentionally absent from this sandbox. Use the source locators only when the verifiable context needs verification.",
+            "Use the local native `interf-query` skill available in this Context Graph.",
+            "Answer the check question the same way you would answer a real user inside this Context Graph.",
+            "Prefer the Build Plan-declared context-graph outputs before consulting source references.",
+            "This sandbox is self-contained: the copied Context Graph has authoritative source references in `.interf/runtime/source-manifest.json`.",
+            "The original Interf instance state is intentionally absent from this sandbox. Use the source locators only when the Context Graph needs verification.",
         ]
         : [
-            "You are running an Interf baseline test inside an isolated source-files test shell.",
+            "You are running an Interf baseline benchmark inside an isolated source-files benchmark shell.",
             "Read `AGENTS.md` first.",
             "Use the local native `interf-query` skill available in this shell.",
-            "There is no verifiable context in this sandbox.",
-            "Answer only from source references listed in `runtime/source-files.json` inside this shell.",
+            "There is no Context Graph in this sandbox.",
+            "Answer only from source references listed in `runtime/source-manifest.json` inside this shell.",
             "The original control plane folder is intentionally absent from this sandbox.",
         ];
     return [
@@ -353,7 +353,7 @@ async function runTestCaseWithJudge(target, testCase, executor) {
     });
     checks.push(...evaluated.checks);
     if (testCase.answer) {
-        const judged = await runTargetTestsJudge(testCase, executor, `verifiable context file ${outputPath}`, content);
+        const judged = await runTargetTestsJudge(testCase, executor, `Context Graph file ${outputPath}`, content);
         checks.push({
             label: "judge verdict",
             ok: judged.verdict?.pass === true,