@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/steps/eval-step.js

@@ -0,0 +1,347 @@
+/**
+ * Pipeline step: Run promptfoo evaluation.
+ *
+ * Preconditions: config files and context files exist
+ * Postconditions: eval-results.json exists and is valid
+ *
+ * Cache key: promptfooconfig*.yaml + contexts/*.md + tasks/*.yaml +
+ *            canonical contexts + reference solutions + config/models.yaml
+ * Cache outputs: results/latest/eval-results*.json
+ *
+ * Remote cache: When local cache misses and a Sanity token is available,
+ * the step queries the Content Lake for a report with a matching eval
+ * fingerprint. On a hit, the cached score-summary.json is written to disk
+ * and the eval + calculate-scores steps are skipped entirely.
+ *
+ * @see docs/design-docs/content-lake-eval-caching.md
+ */
+import { execSync } from "child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
+import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../checks.js";
+import { computeEvalFingerprint } from "../eval-fingerprint.js";
+import { resolveMappings } from "../resolve-mappings.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+const CONFIG_FILES = {
+    agentic: "promptfooconfig.agentic.yaml",
+    baseline: "promptfooconfig.yaml",
+    observed: "promptfooconfig.observed.yaml",
+};
+/** Each mode writes eval results to a different file (set in the config's outputPath) */
+export const RESULTS_FILES = {
+    agentic: "results/latest/eval-results-agentic.json",
+    baseline: "results/latest/eval-results.json",
+    observed: "results/latest/eval-results-observed.json",
+};
+export function buildFilterFlags(debug) {
+    if (!debug?.enabled)
+        return "";
+    const flags = [];
+    if (debug.pattern) {
+        flags.push(`--filter-pattern '${debug.pattern}'`);
+    }
+    if (debug.sample) {
+        flags.push(`--filter-sample ${debug.sample}`);
+    }
+    if (debug.firstN) {
+        flags.push(`--filter-first-n ${debug.firstN}`);
+    }
+    // Default: first 2 tests when no other filters specified
+    if (flags.length === 0) {
+        flags.push("--filter-first-n 2");
+    }
+    return " " + flags.join(" ");
+}
+/**
+ * Extract the Promptfoo share URL from the eval results JSON.
+ *
+ * Promptfoo writes a `shareableUrl` field into the results file when
+ * `PROMPTFOO_API_KEY` is set. This replaces the previous approach of
+ * scraping the URL from a captured log file (which required piping
+ * through `tee` and broke TTY progress reporting).
+ */
+export function extractShareUrl(mode) {
+    const resultsPath = resolve(ROOT, RESULTS_FILES[mode]);
+    if (!existsSync(resultsPath))
+        return undefined;
+    try {
+        const raw = readFileSync(resultsPath, "utf-8");
+        const data = JSON.parse(raw);
+        return data.shareableUrl ?? undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+// ---------------------------------------------------------------------------
+// Post-eval error scanning
+// ---------------------------------------------------------------------------
+export async function runEval(mode, debug, concurrency, noCache = false, remoteCacheOpts) {
+    const start = Date.now();
+    // Precondition: config file exists
+    const configIssues = checkGeneratedConfigsExist(ROOT);
+    const configErrors = configIssues.filter((i) => i.severity === "error");
+    if (configErrors.length > 0) {
+        return {
+            stepResult: {
+                durationMs: Date.now() - start,
+                error: `Config files missing: ${configErrors.map((e) => e.message).join("; ")}`,
+                status: "failed",
+            },
+        };
+    }
+    // Precondition: canonical context files exist for all mapped tasks
+    const mappings = resolveMappings(ROOT);
+    const taskIds = Object.values(mappings.feature_areas).flatMap((area) => area.tasks.map((t) => t.id));
+    const contextIssues = checkCanonicalContextsExist(ROOT, taskIds);
+    const contextErrors = contextIssues.filter((i) => i.severity === "error");
+    if (contextErrors.length > 0) {
+        return {
+            stepResult: {
+                durationMs: Date.now() - start,
+                error: `Context files missing. Run 'pnpm fetch-docs' first. ${contextErrors.map((e) => e.message).join("; ")}`,
+                status: "failed",
+            },
+        };
+    }
+    // -----------------------------------------------------------------------
+    // Compute eval fingerprint (used for both remote cache + provenance)
+    // Only for non-debug runs — debug runs use test subsets.
+    // -----------------------------------------------------------------------
+    let evalFingerprint;
+    if (!debug?.enabled && remoteCacheOpts?.graderModel) {
+        try {
+            evalFingerprint = computeEvalFingerprint({
+                filter: remoteCacheOpts.filter,
+                graderModel: remoteCacheOpts.graderModel,
+                mode,
+                rootDir: ROOT,
+            });
+        }
+        catch (err) {
+            console.warn(`  ⚠️  Could not compute eval fingerprint: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // -----------------------------------------------------------------------
+    // Cache check — local first, then remote
+    // -----------------------------------------------------------------------
+    // Local cache check — skip eval if inputs unchanged (biggest cost saver).
+    // Each mode gets its own cache key so that in `full` mode, a fresh agentic
+    // cache doesn't force baseline to re-run (or vice versa).
+    const cacheKey = `eval-${mode}`;
+    if (!noCache) {
+        const cacheResult = lookupCache(ROOT, cacheKey);
+        if (cacheResult.hit) {
+            return {
+                evalFingerprint,
+                stepResult: {
+                    durationMs: Date.now() - start,
+                    status: "success",
+                    summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
+                },
+            };
+        }
+    }
+    // Remote cache check — query Content Lake for matching fingerprint
+    if (evalFingerprint &&
+        !noCache &&
+        !remoteCacheOpts?.noRemoteCache &&
+        remoteCacheOpts?.sanityToken) {
+        const remoteCacheResult = await checkRemoteCache(evalFingerprint, remoteCacheOpts.sanityToken);
+        if (remoteCacheResult) {
+            return {
+                evalFingerprint,
+                remoteCacheHit: true,
+                stepResult: {
+                    durationMs: Date.now() - start,
+                    status: "success",
+                    summary: `Skipped (remote cache hit) — reusing report ${remoteCacheResult.reportId} from ${remoteCacheResult.completedAt}`,
+                },
+            };
+        }
+    }
+    // Execute — run promptfoo directly with inherited stdio so the TTY
+    // progress bar works in interactive terminals and the CI progress
+    // reporter works in CI environments. Previously this was piped through
+    // `tee` to capture a log file for share-URL extraction, but `tee`
+    // destroyed TTY detection, disabling all progress output. The share URL
+    // is now read from the eval results JSON (`shareableUrl` field) instead.
+    //
+    // Sharing is enabled by default (via PROMPTFOO_API_KEY / cloud config).
+    // We set PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST=1 to prevent promptfoo's
+    // interactive email prompt from blocking the terminal in local TTY
+    // environments. In CI, isCI() already guards against the prompt, but
+    // the env var provides defense-in-depth for all execution contexts.
+    const configFile = CONFIG_FILES[mode];
+    const filterFlags = buildFilterFlags(debug);
+    const concurrencyFlag = concurrency ? ` --max-concurrency ${concurrency}` : "";
+    const noCacheFlag = noCache ? " --no-cache" : "";
+    const evalCmd = `dotenv -e ../../.env -o -- promptfoo eval --config ${configFile}${filterFlags}${concurrencyFlag}${noCacheFlag}`;
+    let exitCode = 0;
+    try {
+        execSync(evalCmd, {
+            cwd: ROOT,
+            env: {
+                ...process.env,
+                PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST: "1",
+            },
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        // promptfoo exits 100 when assertions fail — that's expected, not an error
+        exitCode =
+            err !== null && typeof err === "object" && "status" in err
+                ? err.status
+                : 1;
+        if (exitCode !== 100) {
+            return {
+                evalFingerprint,
+                stepResult: {
+                    durationMs: Date.now() - start,
+                    error: `promptfoo eval failed with exit code ${exitCode}`,
+                    status: "failed",
+                },
+            };
+        }
+    }
+    // Postcondition: results file exists and is valid
+    const resultsIssues = checkResultsExist(ROOT, RESULTS_FILES[mode]);
+    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+    if (resultsErrors.length > 0) {
+        return {
+            evalFingerprint,
+            stepResult: {
+                durationMs: Date.now() - start,
+                error: `Postcondition failed: ${resultsErrors.map((e) => e.message).join("; ")}`,
+                status: "failed",
+            },
+        };
+    }
+    // Scan results for errors and surface them clearly
+    const errorSummary = scanResultsForErrors(resolve(ROOT, RESULTS_FILES[mode]));
+    if (errorSummary) {
+        console.log();
+        console.log(errorSummary);
+    }
+    const durationMs = Date.now() - start;
+    const summary = `Evaluation complete (mode: ${mode}${debug?.enabled ? ", debug" : ""})`;
+    // Record cache — only for non-debug runs (debug uses a subset of tests).
+    // Uses per-mode cache key so baseline and agentic are independently cached.
+    if (!noCache && !debug?.enabled) {
+        const inputPaths = getStepInputPaths(ROOT, cacheKey);
+        const inputHash = hashFiles(inputPaths);
+        recordCache(ROOT, cacheKey, inputHash, summary, durationMs, [
+            RESULTS_FILES[mode],
+        ]);
+    }
+    return {
+        evalFingerprint,
+        stepResult: { durationMs, status: "success", summary },
+    };
+}
+// ---------------------------------------------------------------------------
+// Remote cache helpers
+// ---------------------------------------------------------------------------
+/**
+ * Query the Sanity Content Lake for a report with a matching eval fingerprint.
+ *
+ * On a hit, writes the cached score-summary.json to results/latest/ so that
+ * downstream steps (report, compare, publish) can proceed as if the eval
+ * had just run.
+ *
+ * @returns The matched report metadata on hit, null on miss or error
+ */
+async function checkRemoteCache(fingerprint, sanityToken) {
+    try {
+        const { ReportStore } = await import("../../report-store.js");
+        const store = new ReportStore({
+            dataset: process.env.AILF_REPORT_DATASET ?? undefined,
+            projectId: process.env.AILF_REPORT_PROJECT_ID ?? undefined,
+            token: sanityToken,
+        });
+        const startQuery = Date.now();
+        const cachedReport = await store.findByFingerprint(fingerprint);
+        const queryMs = Date.now() - startQuery;
+        if (!cachedReport) {
+            console.log(`  ℹ️  Remote cache miss — no report matches fingerprint (${queryMs}ms)`);
+            return null;
+        }
+        // Write the cached score summary to disk so downstream steps work
+        const outDir = resolve(ROOT, "results", "latest");
+        if (!existsSync(outDir)) {
+            mkdirSync(outDir, { recursive: true });
+        }
+        writeFileSync(resolve(outDir, "score-summary.json"), JSON.stringify(cachedReport.summary, null, 2));
+        console.log(`  ✅ Remote cache hit — reusing report ${cachedReport.id} from ${cachedReport.completedAt}`);
+        console.log(`  ℹ️  Fingerprint: ${fingerprint.slice(0, 16)}... (${queryMs}ms)`);
+        console.log("  ⚠️  Cached scores are statistically equivalent, not identical");
+        return {
+            completedAt: cachedReport.completedAt,
+            reportId: cachedReport.id,
+        };
+    }
+    catch (err) {
+        console.warn(`  ⚠️  Remote cache check failed: ${err instanceof Error ? err.message : String(err)}`);
+        return null;
+    }
+}
+/**
+ * Read the eval results JSON and produce a human-readable summary of any
+ * errored or failed tests. This surfaces API errors, timeouts, and other
+ * issues that would otherwise be buried in the Promptfoo table output.
+ *
+ * Returns null if there are no errors/failures worth reporting.
+ */
+function scanResultsForErrors(resultsPath) {
+    if (!existsSync(resultsPath))
+        return null;
+    let file;
+    try {
+        const raw = readFileSync(resultsPath, "utf-8");
+        file = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    const results = file?.results?.results;
+    if (!Array.isArray(results))
+        return null;
+    const errored = [];
+    for (const r of results) {
+        if (r.gradingResult !== null)
+            continue;
+        const desc = r.testCase?.description ?? r.description ?? "unknown";
+        const provider = r.provider?.label ?? r.provider?.id ?? "unknown";
+        // No grading result = the provider errored before producing a response.
+        // This is the only case we surface — API 500s, timeouts, rate limits.
+        // Note: r.error may also be set for assertion failures, but those have
+        // a non-null gradingResult and are normal pass/fail outcomes.
+        const errorMsg = r.error
+            ? (typeof r.error === "string" ? r.error : JSON.stringify(r.error)).slice(0, 200)
+            : "Provider returned no scorable result";
+        errored.push({ description: desc, error: errorMsg, provider });
+    }
+    if (errored.length === 0)
+        return null;
+    const total = results.length;
+    const lines = [];
+    lines.push(`  ┌─────────────────────────────────────────────────────────────`);
+    lines.push(`  │ ⚠️  ${errored.length} of ${total} eval result(s) errored (no gradingResult)`);
+    lines.push(`  │`);
+    for (const e of errored) {
+        lines.push(`  │   ✗ [${e.provider}] ${e.description}`);
+        lines.push(`  │     → ${e.error}`);
+    }
+    const errorRate = Math.round((errored.length / total) * 100);
+    if (errorRate >= 25) {
+        lines.push(`  │`);
+        lines.push(`  │ 🔥 High error rate (${errorRate}%) — check API keys, rate limits,`);
+        lines.push(`  │   or model availability. Errored results are excluded from scoring.`);
+    }
+    lines.push(`  └─────────────────────────────────────────────────────────────`);
+    return lines.join("\n");
+}

package/dist/pipeline/steps/fetch-docs-step.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline step: Fetch documentation from Sanity CMS.
+ *
+ * Preconditions: tasks have inline canonical_docs
+ * Postconditions: canonical context files exist for all mapped tasks
+ *
+ * Cache key: tasks/*.yaml + config/sources.yaml + config/models.yaml
+ * Cache outputs: contexts/canonical/*.md files
+ */
+import type { StepResult } from "../types.js";
+export declare function runFetchDocs(source?: string, noCache?: boolean): Promise<StepResult>;

package/dist/pipeline/steps/fetch-docs-step.js

@@ -0,0 +1,84 @@
+/**
+ * Pipeline step: Fetch documentation from Sanity CMS.
+ *
+ * Preconditions: tasks have inline canonical_docs
+ * Postconditions: canonical context files exist for all mapped tasks
+ *
+ * Cache key: tasks/*.yaml + config/sources.yaml + config/models.yaml
+ * Cache outputs: contexts/canonical/*.md files
+ */
+import { execSync } from "child_process";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { lookupCache, recordCache } from "../cache.js";
+import { checkCanonicalContextsExist } from "../checks.js";
+import { resolveMappings } from "../resolve-mappings.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export async function runFetchDocs(source, noCache = false) {
+    const start = Date.now();
+    // Precondition: at least one task has inline canonical mappings
+    const mappings = resolveMappings(ROOT);
+    const totalTasks = Object.values(mappings.feature_areas).reduce((sum, area) => sum + area.tasks.length, 0);
+    if (totalTasks === 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: "No tasks with canonical_docs found in task files. Add canonical_docs to your task definitions.",
+            status: "failed",
+        };
+    }
+    // Cache check
+    if (!noCache) {
+        const cacheResult = lookupCache(ROOT, "fetch-docs");
+        if (cacheResult.hit) {
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
+            };
+        }
+    }
+    // Execute
+    try {
+        const sourceArg = source ? ` --source ${source}` : "";
+        execSync(`pnpm fetch-docs${sourceArg}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `fetch-docs failed: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    // Postcondition: canonical context files exist for all mapped tasks
+    // Re-resolve in case fetch-docs modified things (unlikely but safe)
+    const postMappings = resolveMappings(ROOT);
+    const taskIds = Object.values(postMappings.feature_areas).flatMap((area) => area.tasks.map((t) => t.id));
+    const contextIssues = checkCanonicalContextsExist(ROOT, taskIds);
+    const contextErrors = contextIssues.filter((i) => i.severity === "error");
+    if (contextErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Postcondition failed: ${contextErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    const durationMs = Date.now() - start;
+    const summary = `Fetched canonical contexts for ${taskIds.length} tasks`;
+    // Record cache
+    if (!noCache) {
+        const { getStepInputPaths, hashFiles } = await import("../cache.js");
+        const inputPaths = getStepInputPaths(ROOT, "fetch-docs");
+        const inputHash = hashFiles(inputPaths);
+        const outputPaths = [
+            ...taskIds.map((id) => `contexts/canonical/${id}.md`),
+            "contexts/document-manifest.json",
+        ];
+        recordCache(ROOT, "fetch-docs", inputHash, summary, durationMs, outputPaths);
+    }
+    return { durationMs, status: "success", summary };
+}

package/dist/pipeline/steps/generate-configs-step.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pipeline step: Generate promptfoo config files from config/models.yaml.
+ *
+ * Preconditions: config/models.yaml is valid
+ * Postconditions: promptfooconfig*.yaml files exist
+ *
+ * Cache key: config/models.yaml + config/sources.yaml + tasks/*.yaml
+ * Cache outputs: promptfooconfig*.yaml files
+ */
+import type { StepResult } from "../types.js";
+export declare function runGenerateConfigs(source?: string, noCache?: boolean): StepResult;

package/dist/pipeline/steps/generate-configs-step.js

@@ -0,0 +1,98 @@
+/**
+ * Pipeline step: Generate promptfoo config files from config/models.yaml.
+ *
+ * Preconditions: config/models.yaml is valid
+ * Postconditions: promptfooconfig*.yaml files exist
+ *
+ * Cache key: config/models.yaml + config/sources.yaml + tasks/*.yaml
+ * Cache outputs: promptfooconfig*.yaml files
+ */
+import { execSync } from "child_process";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
+import { checkGeneratedConfigsExist } from "../checks.js";
+import { validateModelsYaml } from "../validate.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+export function runGenerateConfigs(source, noCache = false) {
+    const start = Date.now();
+    // Precondition: config/models.yaml must be valid
+    const modelIssues = validateModelsYaml(ROOT);
+    const errors = modelIssues.filter((i) => i.severity === "error");
+    if (errors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `config/models.yaml validation failed: ${errors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    // Cache check — include filter env vars so scoped runs don't reuse
+    // cached results from unscoped (or differently-scoped) runs.
+    const filterContext = buildFilterContext();
+    if (!noCache) {
+        const cacheResult = lookupCache(ROOT, "generate-configs", filterContext);
+        if (cacheResult.hit) {
+            return {
+                durationMs: Date.now() - start,
+                status: "success",
+                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
+            };
+        }
+    }
+    // Execute
+    try {
+        const sourceArg = source ? ` --source ${source}` : "";
+        execSync(`pnpm generate-configs${sourceArg}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        return {
+            durationMs: Date.now() - start,
+            error: `generate-configs failed: ${err instanceof Error ? err.message : String(err)}`,
+            status: "failed",
+        };
+    }
+    // Postcondition: config files exist
+    const configIssues = checkGeneratedConfigsExist(ROOT);
+    const configErrors = configIssues.filter((i) => i.severity === "error");
+    if (configErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Postcondition failed: ${configErrors.map((e) => e.message).join("; ")}`,
+            status: "failed",
+        };
+    }
+    const durationMs = Date.now() - start;
+    const summary = "Generated promptfoo config files";
+    // Record cache
+    if (!noCache) {
+        const inputPaths = getStepInputPaths(ROOT, "generate-configs");
+        const inputHash = hashFiles(inputPaths, filterContext);
+        const outputPaths = [
+            "promptfooconfig.yaml",
+            "promptfooconfig.observed.yaml",
+            "promptfooconfig.agentic.yaml",
+        ];
+        recordCache(ROOT, "generate-configs", inputHash, summary, durationMs, outputPaths);
+    }
+    return { durationMs, status: "success", summary };
+}
+/**
+ * Build cache context strings from filter environment variables.
+ * When EVAL_FILTER_AREAS or EVAL_FILTER_TASKS are set, they become part
+ * of the cache key so that differently-scoped runs don't share cache entries.
+ */
+function buildFilterContext() {
+    const context = [];
+    if (process.env.EVAL_FILTER_AREAS) {
+        context.push(`areas:${process.env.EVAL_FILTER_AREAS}`);
+    }
+    if (process.env.EVAL_FILTER_TASKS) {
+        context.push(`tasks:${process.env.EVAL_FILTER_TASKS}`);
+    }
+    return context;
+}

package/dist/pipeline/steps/grader-consistency-step.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Pipeline step: Measure grader consistency via replicated grading.
+ *
+ * This step is OPTIONAL — it only runs when --grader-replications N is passed.
+ * It re-runs grading assertions N additional times on the same model responses
+ * and measures score variance across replications.
+ *
+ * Preconditions: eval-results.json exists (model responses to re-grade)
+ * Postconditions: grader-consistency.json written to results/latest/
+ *
+ * Not cached: Each run involves fresh API calls to the grader model.
+ * The whole point is to measure variance, so caching would defeat the purpose.
+ */
+import type { EvalMode, StepResult } from "../types.js";
+/**
+ * Run grader consistency analysis.
+ *
+ * @param replications Number of additional grading replications (default: 5)
+ * @param mode Eval mode — determines which results file to read
+ */
+export declare function runGraderConsistency(replications?: number, mode?: EvalMode): StepResult;

package/dist/pipeline/steps/grader-consistency-step.js

@@ -0,0 +1,74 @@
+/**
+ * Pipeline step: Measure grader consistency via replicated grading.
+ *
+ * This step is OPTIONAL — it only runs when --grader-replications N is passed.
+ * It re-runs grading assertions N additional times on the same model responses
+ * and measures score variance across replications.
+ *
+ * Preconditions: eval-results.json exists (model responses to re-grade)
+ * Postconditions: grader-consistency.json written to results/latest/
+ *
+ * Not cached: Each run involves fresh API calls to the grader model.
+ * The whole point is to measure variance, so caching would defeat the purpose.
+ */
+import { execSync } from "child_process";
+import { existsSync } from "fs";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+import { checkResultsExist } from "../checks.js";
+import { RESULTS_FILES } from "./eval-step.js";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+/**
+ * Run grader consistency analysis.
+ *
+ * @param replications Number of additional grading replications (default: 5)
+ * @param mode Eval mode — determines which results file to read
+ */
+export function runGraderConsistency(replications = 5, mode = "baseline") {
+    const start = Date.now();
+    // For full mode, use baseline results for grader consistency analysis
+    const concreteMode = mode === "full" ? "baseline" : mode;
+    const resultsFile = RESULTS_FILES[concreteMode];
+    const resultsIssues = checkResultsExist(ROOT, resultsFile);
+    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+    if (resultsErrors.length > 0) {
+        return {
+            durationMs: Date.now() - start,
+            error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}. Run eval first.`,
+            status: "failed",
+        };
+    }
+    // Execute
+    try {
+        execSync(`tsx src/lib/grader-consistency.ts --replications ${replications} --results ${resultsFile}`, {
+            cwd: ROOT,
+            env: process.env,
+            stdio: "inherit",
+        });
+    }
+    catch (err) {
+        const code = err !== null && typeof err === "object" && "status" in err
+            ? err.status
+            : 1;
+        return {
+            durationMs: Date.now() - start,
+            error: `grader-consistency failed with exit code ${code}`,
+            status: "failed",
+        };
+    }
+    // Postcondition: output file exists
+    const outputPath = resolve(ROOT, "results", "latest", "grader-consistency.json");
+    if (!existsSync(outputPath)) {
+        return {
+            durationMs: Date.now() - start,
+            error: "grader-consistency.json was not created",
+            status: "failed",
+        };
+    }
+    return {
+        durationMs: Date.now() - start,
+        status: "success",
+        summary: `Grader consistency analysis complete (${replications} replications)`,
+    };
+}