@interf/compiler 0.7.3 → 0.9.3

package/dist/cli/commands/test-flow.js

@@ -1,99 +1,36 @@
 import chalk from "chalk";
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { createRawTestTarget, createCompiledTestTarget, runTargetTestsAuto, saveTargetTestRun, } from "../../packages/testing/test.js";
-import { buildTestSpecFromSourceFolderConfig, buildTestSpecFromCompiledDatasetConfig, fingerprintTruthChecks, resolveSourceDatasetPath, } from "../../packages/project-model/source-config.js";
-import { ensurePortableContextScaffold, } from "../../packages/project-model/interf.js";
-import { datasetLatestTestStatePath, datasetLatestTestSummaryPath, normalizeDatasetTestRunId, datasetTestRunPath, datasetTestRunsRoot, } from "../../packages/project-model/project-paths.js";
-import { testRootForCompiled } from "../../packages/project-model/compiled-paths.js";
-import { readJsonFileWithSchema } from "../../packages/shared/parse.js";
-import { TestRunComparisonSchema } from "../../packages/testing/lib/schema.js";
-import { resolveOrConfigureLocalExecutor } from "./executor-flow.js";
+import { readinessPassRate as readinessPassRateForOutcome, readCurrentSavedReadinessCheckRun, readSavedReadinessCheckRun, runSavedPortableContextCheck, runSavedSourceFilesCheck, saveReadinessCheckRun, } from "../../packages/testing/readiness-check-run.js";
 import { findBuiltCompiledPath } from "./compiled-flow.js";
-export function questionPassRate(outcome) {
-    return outcome.result.totalCases > 0
-        ? Math.round((outcome.result.passedCases / outcome.result.totalCases) * 100)
-        : 0;
+export function readinessPassRate(outcome) {
+    return readinessPassRateForOutcome(outcome);
 }
 function visibleRunPath(outcome) {
     return outcome.displayRunPath ?? outcome.runPath;
 }
-function summarizeSavedTestOutcome(label, outcome) {
-    return {
-        label,
-        run_path: visibleRunPath(outcome),
-        ok: outcome.result.ok,
-        passed_cases: outcome.result.passedCases,
-        total_cases: outcome.result.totalCases,
-        passed_checks: outcome.result.passedChecks,
-        total_checks: outcome.result.totalChecks,
-        target: outcome.target,
-    };
+function loadLatestComparison(projectPath, preparationName) {
+    return readSavedReadinessCheckRun(projectPath, preparationName);
 }
-function writeDatasetTargetRun(options) {
-    const dirPath = datasetTestRunsRoot(options.projectPath, options.datasetName, options.target);
-    mkdirSync(dirPath, { recursive: true });
-    const runPath = datasetTestRunPath(options.projectPath, options.datasetName, options.target, options.generatedAt, options.runId, options.runSuffix);
-    writeFileSync(runPath, `${JSON.stringify(options.payload, null, 2)}\n`);
-    return runPath;
-}
-function loadLatestComparison(projectPath, datasetName) {
-    const latestPath = datasetLatestTestStatePath(projectPath, datasetName);
-    if (!existsSync(latestPath))
-        return null;
-    return readJsonFileWithSchema(latestPath, "latest test comparison", TestRunComparisonSchema);
-}
-export function readSavedTestComparison(projectPath, datasetName) {
-    return loadLatestComparison(projectPath, datasetName);
+export function readSavedTestComparison(projectPath, preparationName) {
+    return loadLatestComparison(projectPath, preparationName);
 }
 export function readCurrentSavedTestComparison(options) {
-    const latestComparison = readSavedTestComparison(options.projectPath, options.datasetName);
-    if (!latestComparison) {
-        return { comparison: null, stale: false };
-    }
-    const currentFingerprint = fingerprintTruthChecks(options.checks);
-    if (!latestComparison.checks_fingerprint || latestComparison.checks_fingerprint !== currentFingerprint) {
-        return { comparison: null, stale: true };
-    }
-    return { comparison: latestComparison, stale: false };
-}
-function renderLatestSummaryMarkdown(payload) {
-    const lines = [
-        "# Latest Test Results",
-        "",
-        "| Target | Questions |",
-        "| --- | --- |",
-    ];
-    if (payload.raw) {
-        lines.push(`| Source files | \`${payload.raw.passed_cases}/${payload.raw.total_cases}\` |`);
-    }
-    if (payload.compiled) {
-        lines.push(`| Portable context | \`${payload.compiled.passed_cases}/${payload.compiled.total_cases}\` |`);
-    }
-    lines.push("");
-    if (payload.summary.raw_pass_rate != null && payload.summary.compiled_pass_rate != null) {
-        const direction = (payload.summary.pass_rate_delta ?? 0) >= 0 ? "improved" : "decreased";
-        lines.push(`Question pass rate ${direction} from ${payload.summary.raw_pass_rate}% to ${payload.summary.compiled_pass_rate}%.`, "");
-    }
-    if (payload.raw) {
-        lines.push(`- Latest source-files run: ${payload.raw.run_path}`);
-    }
-    if (payload.compiled) {
-        lines.push(`- Latest portable-context run: ${payload.compiled.run_path}`);
-    }
-    return `${lines.join("\n")}\n`;
+    return readCurrentSavedReadinessCheckRun({
+        projectPath: options.projectPath,
+        preparationName: options.preparationName,
+        checks: options.checks,
+    });
 }
 export function printSavedTestComparisonState(payload, comparisonRunPath) {
     console.log();
-    console.log(chalk.bold("  Latest saved questions"));
+    console.log(chalk.bold("  Latest readiness results"));
     console.log();
-    console.log("  | Target | Questions |");
+    console.log("  | Target | Readiness checks |");
     console.log("  | --- | --- |");
     if (payload.raw) {
         console.log(`  | Source files | \`${payload.raw.passed_cases}/${payload.raw.total_cases}\` |`);
     }
     if (payload.compiled) {
-        console.log(`  | Portable context | \`${payload.compiled.passed_cases}/${payload.compiled.total_cases}\` |`);
+        console.log(`  | Portable Context | \`${payload.compiled.passed_cases}/${payload.compiled.total_cases}\` |`);
     }
     if (!payload.raw || !payload.compiled) {
         console.log();
@@ -108,7 +45,7 @@ export function printSavedTestComparisonState(payload, comparisonRunPath) {
         const direction = (payload.summary.pass_rate_delta ?? 0) >= 0 ? "improved" : "decreased";
         const color = (payload.summary.pass_rate_delta ?? 0) >= 0 ? chalk.green : chalk.red;
         console.log();
-        console.log(color(`  Question pass rate ${direction} from ${payload.summary.raw_pass_rate}% to ${payload.summary.compiled_pass_rate}%.`));
+        console.log(color(`  Readiness-check pass rate ${direction} from ${payload.summary.raw_pass_rate}% to ${payload.summary.compiled_pass_rate}%.`));
     }
     if (comparisonRunPath) {
         console.log();
@@ -118,7 +55,7 @@ export function printSavedTestComparisonState(payload, comparisonRunPath) {
 function padCell(value, width) {
     return value.padEnd(width, " ");
 }
-function scoreCell(outcome) {
+function resultCell(outcome) {
     if (!outcome)
         return "—";
     return `${outcome.result.passedCases}/${outcome.result.totalCases}`;
@@ -138,13 +75,13 @@ export function printAgentTestMatrix(rows) {
     const headers = [
         "Agent",
         ...(includeRaw ? ["Source files"] : []),
-        ...(includeCompiled ? ["Portable context"] : []),
+        ...(includeCompiled ? ["Portable Context"] : []),
         ...(includeDelta ? ["Delta"] : []),
     ];
     const body = rows.map((row) => [
         row.agentLabel,
-        ...(includeRaw ? [scoreCell(row.rawOutcome)] : []),
-        ...(includeCompiled ? [scoreCell(row.compiledOutcome)] : []),
+        ...(includeRaw ? [resultCell(row.rawOutcome)] : []),
+        ...(includeCompiled ? [resultCell(row.compiledOutcome)] : []),
         ...(includeDelta ? [deltaCell(row)] : []),
     ]);
     const widths = headers.map((header, index) => Math.max(header.length, ...body.map((row) => (row[index] ?? "").length)));
@@ -163,7 +100,7 @@ export function printAgentTestFailures(rows) {
         const failures = [];
         for (const [label, outcome] of [
             ["Source files", row.rawOutcome ?? null],
-            ["Portable context", row.compiledOutcome ?? null],
+            ["Portable Context", row.compiledOutcome ?? null],
         ]) {
             if (!outcome || outcome.result.ok)
                 continue;
@@ -171,7 +108,7 @@ export function printAgentTestFailures(rows) {
                 if (caseResult.ok)
                     continue;
                 const reason = caseResult.checks.find((entry) => !entry.ok)?.detail ?? "failed";
-                failures.push(`${label} · Question ${index + 1}: ${reason}`);
+                failures.push(`${label} · Check ${index + 1}: ${reason}`);
             }
         }
         if (failures.length === 0)
@@ -184,32 +121,16 @@ export function printAgentTestFailures(rows) {
         }
     }
 }
-function specNeedsExecutor(spec) {
-    return spec.cases.some((entry) => !entry.file || Boolean(entry.answer));
-}
-async function resolveExecutorForSpec(spec, executor, executionProfile) {
-    if (!specNeedsExecutor(spec)) {
-        return { executor: null, error: undefined };
-    }
-    if (executor) {
-        return { executor, error: undefined };
-    }
-    return resolveOrConfigureLocalExecutor({
-        preflight: true,
-        executionProfile,
-        purpose: "test",
-    });
-}
 export function printSavedTestOutcome(prefix, outcome) {
     const color = outcome.result.ok ? chalk.green : chalk.red;
     const status = outcome.result.ok ? chalk.green("PASS") : chalk.red("FAIL");
     console.log(chalk.bold(`  ${prefix}`));
     console.log();
-    console.log(color(`  ${status} ${outcome.result.passedCases}/${outcome.result.totalCases} tests`));
+    console.log(color(`  ${status} ${outcome.result.passedCases}/${outcome.result.totalCases} readiness checks`));
     console.log();
     for (const [index, caseResult] of outcome.result.caseResults.entries()) {
         const caseStatus = caseResult.ok ? chalk.green("PASS") : chalk.red("FAIL");
-        console.log(`  ${caseStatus} Question ${index + 1}`);
+        console.log(`  ${caseStatus} Check ${index + 1}`);
         console.log(chalk.dim(`       ${caseResult.question}`));
         const failedChecks = caseResult.checks.filter((entry) => !entry.ok).slice(0, 3);
         for (const failedCheck of failedChecks) {
@@ -233,16 +154,16 @@ export function printSavedTestComparison(rawOutcome, compiledOutcome, comparison
     if (compiledOutcome) {
         if (rawOutcome)
             console.log();
-        printSavedTestOutcome("Portable context", compiledOutcome);
+        printSavedTestOutcome("Portable Context", compiledOutcome);
     }
     if (rawOutcome && compiledOutcome) {
-        const rawQuestions = questionPassRate(rawOutcome);
-        const compiledQuestions = questionPassRate(compiledOutcome);
-        const delta = compiledQuestions - rawQuestions;
+        const rawReadiness = readinessPassRate(rawOutcome);
+        const compiledReadiness = readinessPassRate(compiledOutcome);
+        const delta = compiledReadiness - rawReadiness;
         const color = delta >= 0 ? chalk.green : chalk.red;
         const direction = delta >= 0 ? "improved" : "decreased";
         console.log();
-        console.log(color(`  Question pass rate ${direction} from ${rawQuestions}% to ${compiledQuestions}%.`));
+        console.log(color(`  Readiness-check pass rate ${direction} from ${rawReadiness}% to ${compiledReadiness}%.`));
     }
     if (comparisonRunPath) {
         console.log();
@@ -255,22 +176,22 @@ export function printSavedTestComparisonSummary(rawOutcome, compiledOutcome, com
     console.log();
     console.log(chalk.bold("  Comparison"));
     console.log();
-    console.log("  | Target | Questions |");
+    console.log("  | Target | Readiness checks |");
     console.log("  | --- | --- |");
     if (rawOutcome) {
         console.log(`  | Source files | \`${rawOutcome.result.passedCases}/${rawOutcome.result.totalCases}\` |`);
     }
     if (compiledOutcome) {
-        console.log(`  | Portable context | \`${compiledOutcome.result.passedCases}/${compiledOutcome.result.totalCases}\` |`);
+        console.log(`  | Portable Context | \`${compiledOutcome.result.passedCases}/${compiledOutcome.result.totalCases}\` |`);
     }
     if (rawOutcome && compiledOutcome) {
-        const rawQuestions = questionPassRate(rawOutcome);
-        const compiledQuestions = questionPassRate(compiledOutcome);
-        const delta = compiledQuestions - rawQuestions;
+        const rawReadiness = readinessPassRate(rawOutcome);
+        const compiledReadiness = readinessPassRate(compiledOutcome);
+        const delta = compiledReadiness - rawReadiness;
         const color = delta >= 0 ? chalk.green : chalk.red;
         const direction = delta >= 0 ? "improved" : "decreased";
         console.log();
-        console.log(color(`  Question pass rate ${direction} from ${rawQuestions}% to ${compiledQuestions}%.`));
+        console.log(color(`  Readiness-check pass rate ${direction} from ${rawReadiness}% to ${compiledReadiness}%.`));
     }
     if (comparisonRunPath) {
         console.log();
@@ -278,144 +199,33 @@ export function printSavedTestComparisonSummary(rawOutcome, compiledOutcome, com
     }
 }
 export function saveTestComparisonRun(options) {
-    const generatedAt = new Date().toISOString();
-    const rawSummary = options.rawOutcome
-        ? summarizeSavedTestOutcome("Source files", options.rawOutcome)
-        : null;
-    const compiledSummary = options.compiledOutcome
-        ? summarizeSavedTestOutcome("Portable context", options.compiledOutcome)
-        : null;
-    const effectiveMode = rawSummary && compiledSummary
-        ? "both"
-        : rawSummary
-            ? "raw"
-            : "compiled";
-    const rawPassRate = rawSummary
-        ? Math.round((rawSummary.passed_cases / rawSummary.total_cases) * 100)
-        : null;
-    const compiledPassRate = compiledSummary
-        ? Math.round((compiledSummary.passed_cases / compiledSummary.total_cases) * 100)
-        : null;
-    const payload = {
-        kind: "interf-test-run",
-        version: 1,
-        generated_at: generatedAt,
-        mode: effectiveMode,
-        source_path: options.sourcePath,
-        checks_fingerprint: options.checksFingerprint,
-        dataset: {
-            name: options.compiledName,
-            compiled_path: options.compiledPath,
-        },
-        raw: rawSummary,
-        compiled: compiledSummary,
-        summary: {
-            raw_pass_rate: rawPassRate,
-            compiled_pass_rate: compiledPassRate,
-            pass_rate_delta: rawPassRate !== null && compiledPassRate !== null ? compiledPassRate - rawPassRate : null,
-        },
-    };
-    const latestStatePath = datasetLatestTestStatePath(options.sourcePath, options.compiledName);
-    mkdirSync(dirname(latestStatePath), { recursive: true });
-    writeFileSync(latestStatePath, `${JSON.stringify(payload, null, 2)}\n`);
-    writeFileSync(datasetLatestTestSummaryPath(options.sourcePath, options.compiledName), renderLatestSummaryMarkdown(payload));
-    if (options.compiledPath) {
-        mkdirSync(testRootForCompiled(options.compiledPath), { recursive: true });
-        writeFileSync(join(testRootForCompiled(options.compiledPath), "latest.json"), `${JSON.stringify(payload, null, 2)}\n`);
-    }
-    return latestStatePath;
+    return saveReadinessCheckRun({
+        sourcePath: options.sourcePath,
+        portableContextPath: options.compiledPath,
+        preparationName: options.compiledName,
+        checksFingerprint: options.checksFingerprint,
+        mode: options.mode,
+        rawOutcome: options.rawOutcome,
+        compiledOutcome: options.compiledOutcome,
+    });
 }
 export async function runSavedRawTest(options) {
-    const spec = buildTestSpecFromSourceFolderConfig({
+    return runSavedSourceFilesCheck({
         sourcePath: options.sourcePath,
-        targetName: options.datasetConfig.name,
-        targetType: "raw",
-    });
-    if (!spec) {
-        return null;
-    }
-    const { executor, error } = await resolveExecutorForSpec(spec, options.executor, options.executionProfile);
-    if (!executor && !error) {
-        return null;
-    }
-    if (!executor && error) {
-        process.exitCode = 1;
-        console.log(chalk.red(error));
-        return null;
-    }
-    const datasetSourcePath = resolveSourceDatasetPath(options.sourcePath, options.datasetConfig);
-    const portableContextPath = ensurePortableContextScaffold(options.sourcePath, options.datasetConfig.name, options.datasetConfig.workflow ?? "interf");
-    const target = createRawTestTarget(datasetSourcePath);
-    const run = await runTargetTestsAuto(datasetSourcePath, spec, [target], {
-        executor,
+        preparationConfig: options.preparationConfig,
+        executor: options.executor,
         preserveSandboxes: options.preserveSandboxes ?? "on-failure",
-        artifactRootPath: testRootForCompiled(portableContextPath),
-    });
-    const result = run.results[0];
-    if (!result)
-        return null;
-    const datasetRunPath = writeDatasetTargetRun({
-        projectPath: options.sourcePath,
-        datasetName: options.datasetConfig.name,
-        target: "file-as-is",
-        generatedAt: run.generated_at,
-        runId: normalizeDatasetTestRunId(spec.id),
         runSuffix: options.runSuffix,
-        payload: run,
     });
-    return {
-        runPath: datasetRunPath,
-        target,
-        result,
-    };
 }
 export async function runSavedCompiledTest(options) {
-    const compiledPath = options.compiledPath ?? findBuiltCompiledPath(options.sourcePath, options.datasetConfig.name);
-    if (!compiledPath) {
-        return null;
-    }
-    const spec = buildTestSpecFromCompiledDatasetConfig({
-        compiledPath,
-        targetType: "compiled",
-    });
-    if (!spec) {
-        return null;
-    }
-    const target = createCompiledTestTarget(compiledPath, options.datasetConfig.name, options.datasetConfig.workflow ?? "interf");
-    if (!target || !target.eligible) {
-        return null;
-    }
-    const { executor, error } = await resolveExecutorForSpec(spec, options.executor, options.executionProfile);
-    if (!executor && !error) {
-        return null;
-    }
-    if (!executor && error) {
-        process.exitCode = 1;
-        console.log(chalk.red(error));
-        return null;
-    }
-    const run = await runTargetTestsAuto(options.sourcePath, spec, [target], {
-        executor,
+    const compiledPath = options.compiledPath ?? findBuiltCompiledPath(options.sourcePath, options.preparationConfig.name);
+    return runSavedPortableContextCheck({
+        sourcePath: options.sourcePath,
+        preparationConfig: options.preparationConfig,
+        executor: options.executor,
+        portableContextPath: compiledPath,
         preserveSandboxes: options.preserveSandboxes ?? "on-failure",
-        artifactRootPath: compiledPath,
-    });
-    const result = run.results[0];
-    if (!result)
-        return null;
-    const compiledRunPath = saveTargetTestRun(compiledPath, run);
-    const datasetRunPath = writeDatasetTargetRun({
-        projectPath: options.sourcePath,
-        datasetName: options.datasetConfig.name,
-        target: "compiled",
-        generatedAt: run.generated_at,
-        runId: normalizeDatasetTestRunId(spec.id),
         runSuffix: options.runSuffix,
-        payload: run,
     });
-    return {
-        runPath: compiledRunPath,
-        displayRunPath: datasetRunPath,
-        target,
-        result,
-    };
 }

package/dist/cli/commands/test.d.ts

@@ -1,14 +1,18 @@
 import type { CommandModule } from "yargs";
-import type { SourceDatasetConfig } from "../../packages/project-model/lib/schema.js";
+import type { SourcePreparationConfig } from "../../packages/project-model/lib/schema.js";
 import { type AgentTestMatrixRow } from "./test-flow.js";
 type TestMode = "raw" | "compiled" | "both";
 export interface TestCommandResult {
     sourcePath: string;
-    datasetConfig: SourceDatasetConfig;
+    preparationConfig: SourcePreparationConfig;
     builtCompiledPath: string | null;
     mode: TestMode;
     rows: AgentTestMatrixRow[];
 }
+export interface TestCommandFailure {
+    message: string;
+    hints: string[];
+}
 export declare const testCommand: CommandModule;
 export declare function runTestCommand(argv?: Record<string, unknown>): Promise<boolean>;
 export {};