@sanity/ailf 0.1.0

package/dist/pipeline/degradations.js

@@ -0,0 +1,242 @@
+/**
+ * pipeline/degradations.ts
+ *
+ * Programmatic code degradation strategies for sensitivity testing.
+ *
+ * Each strategy takes a "good" reference solution (string) and returns a
+ * "bad" version that should score lower on a specific dimension:
+ *
+ * - Task Completion: remove key functional sections
+ * - Code Correctness: introduce anti-patterns and deprecated APIs
+ * - Doc Coverage: strip documentation references, add hallucinated details
+ *
+ * These are deterministic, pure functions — no randomness, no side effects.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+// ---------------------------------------------------------------------------
+// Task Completion degradations
+// ---------------------------------------------------------------------------
+/**
+ * Remove the bottom half of the code (functions, exports, etc).
+ * A response missing half its functionality should score lower on Task Completion.
+ */
+export const removeBottomHalf = {
+    apply(source) {
+        const lines = source.split("\n");
+        const midpoint = Math.floor(lines.length / 2);
+        const kept = lines.slice(0, midpoint);
+        kept.push("");
+        kept.push("// ... (remaining implementation not provided)");
+        return kept.join("\n");
+    },
+    description: "Remove bottom half of code — missing key functionality",
+    targetDimension: "taskCompletion",
+};
+/**
+ * Remove all export statements and exported functions.
+ * Missing exports = incomplete API surface → lower Task Completion.
+ */
+export const removeExports = {
+    apply(source) {
+        const lines = source.split("\n");
+        return lines
+            .filter((line) => {
+            const trimmed = line.trim();
+            // Remove export declarations
+            if (trimmed.startsWith("export "))
+                return false;
+            // Remove lines that are just "export {"
+            if (/^export\s*\{/.test(trimmed))
+                return false;
+            return true;
+        })
+            .join("\n");
+    },
+    description: "Remove export statements — incomplete public API",
+    targetDimension: "taskCompletion",
+};
+/**
+ * Replace function bodies with TODO comments.
+ * Skeleton code that doesn't actually implement anything.
+ */
+export const stubFunctions = {
+    apply(source) {
+        // Match function/method declarations and replace their bodies
+        // This handles: async function foo() { ... }, function foo() { ... },
+        // const foo = async () => { ... }, etc.
+        return source.replace(/(\{)\s*\n([\s\S]*?)(\n\s*\})/g, (match, open, _body, close) => {
+            // Only replace if the body has more than 2 lines (avoid replacing object literals)
+            const bodyLines = _body.split("\n").filter((l) => l.trim().length > 0);
+            if (bodyLines.length < 3)
+                return match;
+            return `${open}\n  // TODO: implement\n  throw new Error("Not implemented")\n${close}`;
+        });
+    },
+    description: "Replace function bodies with TODO stubs — no implementation",
+    targetDimension: "taskCompletion",
+};
+// ---------------------------------------------------------------------------
+// Code Correctness degradations
+// ---------------------------------------------------------------------------
+/**
+ * Replace modern API calls with deprecated/incorrect patterns.
+ * Targets Sanity-specific patterns that the grader should catch.
+ */
+export const introduceDeprecatedAPIs = {
+    apply(source) {
+        let result = source;
+        // Replace createClient with deprecated @sanity/client import
+        result = result.replace(/import\s*\{\s*createClient\s*\}\s*from\s*["']@sanity\/client["']/g, 'import sanityClient from "@sanity/client" // deprecated default import');
+        result = result.replace(/import\s*\{\s*createClient\s*\}\s*from\s*["']next-sanity["']/g, 'import sanityClient from "next-sanity" // deprecated default import');
+        // Replace defineType/defineField with plain objects
+        result = result.replace(/import\s*\{[^}]*defineType[^}]*\}\s*from\s*["']sanity["']/g, "// Using plain objects instead of defineType/defineField");
+        result = result.replace(/defineType\(\{/g, "({");
+        result = result.replace(/defineField\(\{/g, "({");
+        // Replace apiVersion with very old version
+        result = result.replace(/apiVersion:\s*["'][^"']+["']/g, 'apiVersion: "2021-03-25" // outdated API version');
+        // Replace useCdn: true with useCdn: false (incorrect for read-only)
+        result = result.replace(/useCdn:\s*true/g, "useCdn: false // unnecessary");
+        return result;
+    },
+    description: "Replace modern APIs with deprecated patterns",
+    targetDimension: "codeCorrectness",
+};
+/**
+ * Introduce common GROQ syntax errors.
+ * Invalid queries that look plausible but wouldn't work.
+ */
+export const introduceGroqErrors = {
+    apply(source) {
+        let result = source;
+        // Replace -> dereference with . (incorrect)
+        result = result.replace(/->/g, ".");
+        // Replace valid array slice [0...10] with incorrect [0:10]
+        result = result.replace(/\[(\d+)\.\.\.(\d+)\]/g, "[$1:$2]");
+        // Replace references() with invalid refs()
+        result = result.replace(/references\(/g, "refs(");
+        // Replace | order() with incorrect .sort()
+        result = result.replace(/\|\s*order\(/g, ".sort(");
+        return result;
+    },
+    description: "Introduce GROQ syntax errors — plausible but broken queries",
+    targetDimension: "codeCorrectness",
+};
+/**
+ * Remove all TypeScript types and use any everywhere.
+ * Technically works but is an anti-pattern.
+ */
+export const removeTypes = {
+    apply(source) {
+        let result = source;
+        // Remove interface/type declarations
+        result = result.replace(/^(?:export\s+)?(?:interface|type)\s+\w+[\s\S]*?^\}/gm, "");
+        // Replace typed parameters with any
+        result = result.replace(/:\s*[A-Z]\w+(?:\[\])?(?:\s*\|[^,)]+)?/g, ": any");
+        // Replace Promise<Type> with Promise<any>
+        result = result.replace(/Promise<[^>]+>/g, "Promise<any>");
+        // Remove generic type parameters
+        result = result.replace(/<[A-Z]\w+(?:\[\])?>/g, "");
+        return result;
+    },
+    description: "Strip TypeScript types and use 'any' — works but anti-pattern",
+    targetDimension: "codeCorrectness",
+};
+// ---------------------------------------------------------------------------
+// Doc Coverage degradations
+// ---------------------------------------------------------------------------
+/**
+ * Remove all comments and documentation.
+ * The response shows no evidence of using documentation.
+ */
+export const stripComments = {
+    apply(source) {
+        const lines = source.split("\n");
+        return (lines
+            .filter((line) => {
+            const trimmed = line.trim();
+            // Remove single-line comments
+            if (trimmed.startsWith("//"))
+                return false;
+            // Remove JSDoc comment blocks
+            if (trimmed.startsWith("*") ||
+                trimmed.startsWith("/**") ||
+                trimmed.startsWith("*/"))
+                return false;
+            return true;
+        })
+            .join("\n")
+            // Remove remaining inline comments
+            .replace(/\s*\/\/[^"']*$/gm, "")
+            // Remove multi-line comments
+            .replace(/\/\*[\s\S]*?\*\//g, "")
+            // Clean up excessive blank lines
+            .replace(/\n{3,}/g, "\n\n"));
+    },
+    description: "Remove all comments — no evidence of documentation usage",
+    targetDimension: "docCoverage",
+};
+/**
+ * Add hallucinated API calls and made-up configuration options.
+ * Response looks confident but uses APIs that don't exist.
+ */
+export const addHallucinations = {
+    apply(source) {
+        const hallucinations = [
+            "\n// Enable real-time sync (recommended for production)",
+            'const sync = client.enableRealTimeSync({ mode: "aggressive" })',
+            "",
+            "// Configure auto-indexing for faster queries",
+            'client.configureIndex({ fields: ["title", "slug"], type: "fulltext" })',
+            "",
+            "// Use built-in caching middleware",
+            'import { withSanityCache } from "@sanity/cache-middleware"',
+            "const cachedClient = withSanityCache(client, { ttl: 3600 })",
+            "",
+        ];
+        return hallucinations.join("\n") + "\n" + source;
+    },
+    description: "Add hallucinated APIs — confident but wrong information",
+    targetDimension: "docCoverage",
+};
+/**
+ * Replace correct import paths with plausible but wrong ones.
+ * Shows the response was guessing at the API surface.
+ */
+export const wrongImports = {
+    apply(source) {
+        let result = source;
+        // Replace real packages with plausible wrong ones
+        result = result.replace(/from\s*["']@sanity\/client["']/g, 'from "sanity-client"');
+        result = result.replace(/from\s*["']next-sanity["']/g, 'from "@next/sanity"');
+        result = result.replace(/from\s*["']sanity["']/g, 'from "@sanity/core"');
+        result = result.replace(/from\s*["']sanity\/presentation["']/g, 'from "@sanity/presentation-tool"');
+        result = result.replace(/from\s*["']sanity\/structure["']/g, 'from "@sanity/desk-tool"');
+        result = result.replace(/from\s*["']sanity\/functions["']/g, 'from "@sanity/serverless"');
+        return result;
+    },
+    description: "Replace imports with wrong packages — guessing at API surface",
+    targetDimension: "docCoverage",
+};
+// ---------------------------------------------------------------------------
+// Strategy registry
+// ---------------------------------------------------------------------------
+/** All available degradation strategies, grouped by target dimension */
+export const DEGRADATION_STRATEGIES = [
+    // Task Completion
+    removeBottomHalf,
+    removeExports,
+    stubFunctions,
+    // Code Correctness
+    introduceDeprecatedAPIs,
+    introduceGroqErrors,
+    removeTypes,
+    // Doc Coverage
+    stripComments,
+    addHallucinations,
+    wrongImports,
+];
+/** Get degradation strategies targeting a specific dimension */
+export function getStrategiesForDimension(dimension) {
+    return DEGRADATION_STRATEGIES.filter((s) => s.targetDimension === dimension);
+}

package/dist/pipeline/discovery-report.d.ts

@@ -0,0 +1,55 @@
+/**
+ * pipeline/discovery-report.ts
+ *
+ * Generates an agent discoverability report from agentic mode retrieval
+ * metrics. Reads score-summary.json (which contains `retrievalMetrics`
+ * from agentic evaluation) and produces a markdown report showing:
+ *
+ *   - Retrieval summary (recall, precision, F1)
+ *   - Per-area retrieval breakdown
+ *   - Invisible documents (never retrieved by any task)
+ *   - Recommendations for improving discoverability
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks.
+ *
+ * Phase 5c of the Scenario Matrix implementation (Scenarios 4.1 and 4.2).
+ *
+ * @see docs/design-docs/retrieval-metrics.md
+ */
+import type { AreaRetrievalMetrics, RetrievalMetrics, ScoreSummary } from "./types.js";
+export interface DiscoveryReport {
+    /** All areas included in the report (after filtering) */
+    areas: AreaRetrievalMetrics[];
+    /** Base URL from the score summary source config */
+    baseUrl: string | undefined;
+    /** Document slugs that were never retrieved by any task */
+    invisibleDocs: InvisibleDoc[];
+    /** Overall retrieval metrics */
+    overall: RetrievalMetrics["overall"];
+    /** Actionable recommendations */
+    recommendations: string[];
+    /** ISO timestamp of the source evaluation */
+    timestamp: string;
+    /** Total canonical docs across included areas */
+    totalCanonicalDocs: number;
+    /** Total hits (canonical docs successfully retrieved) */
+    totalHits: number;
+}
+export interface InvisibleDoc {
+    /** Tasks that reference this document via canonical_docs */
+    referencedBy: string[];
+    /** The document slug */
+    slug: string;
+}
+/**
+ * Format a discovery report as markdown.
+ */
+export declare function formatDiscoveryMarkdown(report: DiscoveryReport): string;
+/**
+ * Generate a structured discovery report from a score summary.
+ *
+ * @param summary - Parsed score-summary.json
+ * @param areaFilter - Optional area names to include (all if empty)
+ */
+export declare function generateDiscoveryReport(summary: ScoreSummary, areaFilter?: string[]): DiscoveryReport;

package/dist/pipeline/discovery-report.js

@@ -0,0 +1,178 @@
+// ---------------------------------------------------------------------------
+// Core logic (exported for testing)
+// ---------------------------------------------------------------------------
+/**
+ * Format a discovery report as markdown.
+ */
+export function formatDiscoveryMarkdown(report) {
+    const lines = [];
+    // Header
+    lines.push("## 🔍 Agent Discoverability Report");
+    lines.push("");
+    if (report.baseUrl) {
+        lines.push(`**Base URL:** ${report.baseUrl}`);
+    }
+    lines.push("**Mode:** Agentic");
+    lines.push("");
+    // Retrieval summary table
+    lines.push("### Retrieval Summary");
+    lines.push("");
+    lines.push("| Metric | Value |");
+    lines.push("|---|---|");
+    lines.push(`| Recall (canonical docs found) | ${pct(report.overall.avgRecall)} (${report.totalHits}/${report.totalCanonicalDocs}) |`);
+    lines.push(`| Precision (relevant docs fetched) | ${pct(report.overall.avgPrecision)} |`);
+    lines.push(`| F1 Score | ${report.overall.avgF1.toFixed(2)} |`);
+    lines.push(`| Invisible docs | ${report.invisibleDocs.length} |`);
+    lines.push("");
+    // Per-area breakdown
+    if (report.areas.length > 0) {
+        lines.push("### Per-Area Breakdown");
+        lines.push("");
+        lines.push("| Area | Recall | Precision | F1 | Tasks |");
+        lines.push("|---|---|---|---|---|");
+        for (const area of sortedAreas(report.areas)) {
+            lines.push(`| ${area.area} | ${pct(area.avgRecall)} | ${pct(area.avgPrecision)} | ${area.avgF1.toFixed(2)} | ${area.taskCount} |`);
+        }
+        lines.push("");
+    }
+    // Invisible documents
+    if (report.invisibleDocs.length > 0) {
+        lines.push("### Invisible Documents (never retrieved by any task)");
+        lines.push("");
+        for (const doc of report.invisibleDocs) {
+            const refs = doc.referencedBy.join(", ");
+            lines.push(`- \`${doc.slug}\` — referenced by ${refs}`);
+        }
+        lines.push("");
+    }
+    // Recommendations
+    if (report.recommendations.length > 0) {
+        lines.push("### Recommendations");
+        lines.push("");
+        for (let i = 0; i < report.recommendations.length; i++) {
+            lines.push(`${i + 1}. ${report.recommendations[i]}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Generate a structured discovery report from a score summary.
+ *
+ * @param summary - Parsed score-summary.json
+ * @param areaFilter - Optional area names to include (all if empty)
+ */
+export function generateDiscoveryReport(summary, areaFilter) {
+    const metrics = summary.retrievalMetrics;
+    if (!metrics) {
+        throw new Error("score-summary.json does not contain retrievalMetrics. " +
+            "Run an agentic evaluation first: pnpm pipeline -- --mode agentic");
+    }
+    // Apply area filter
+    const areas = areaFilter && areaFilter.length > 0
+        ? metrics.areas.filter((a) => areaFilter.includes(a.area))
+        : metrics.areas;
+    if (areaFilter && areaFilter.length > 0 && areas.length === 0) {
+        throw new Error(`No retrieval data found for area(s): ${areaFilter.join(", ")}. ` +
+            `Available areas: ${metrics.areas.map((a) => a.area).join(", ")}`);
+    }
+    // Recompute overall metrics for filtered areas
+    const overall = areas.length === metrics.areas.length
+        ? metrics.overall
+        : computeOverall(areas);
+    // Build invisible docs list with task references
+    const invisibleDocs = buildInvisibleDocs(areas);
+    // Compute totals for the summary table
+    const allTasks = areas.flatMap((a) => a.tasks);
+    const allExpected = new Set(allTasks.flatMap((t) => t.expected));
+    const allHits = new Set(allTasks.flatMap((t) => t.hits));
+    const totalCanonicalDocs = allExpected.size;
+    const totalHits = allHits.size;
+    // Generate recommendations
+    const recommendations = generateRecommendations(invisibleDocs, areas, overall);
+    return {
+        areas,
+        baseUrl: summary.source?.baseUrl,
+        invisibleDocs,
+        overall,
+        recommendations,
+        timestamp: summary.timestamp,
+        totalCanonicalDocs,
+        totalHits,
+    };
+}
+// ---------------------------------------------------------------------------
+// Helpers (alphabetical for perfectionist/sort-modules)
+// ---------------------------------------------------------------------------
+function buildInvisibleDocs(areas) {
+    // Collect all invisible slugs and map them to the tasks that reference them
+    const slugToTasks = new Map();
+    for (const area of areas) {
+        for (const task of area.tasks) {
+            for (const slug of task.missed) {
+                // Check if this slug is globally invisible (never retrieved by ANY task)
+                const isGloballyInvisible = areas.every((a) => a.tasks.every((t) => !t.retrieved.includes(slug)));
+                if (isGloballyInvisible) {
+                    if (!slugToTasks.has(slug)) {
+                        slugToTasks.set(slug, new Set());
+                    }
+                    slugToTasks.get(slug).add(task.taskId);
+                }
+            }
+        }
+    }
+    return [...slugToTasks.entries()]
+        .map(([slug, tasks]) => ({
+        referencedBy: [...tasks].sort(),
+        slug,
+    }))
+        .sort((a, b) => b.referencedBy.length - a.referencedBy.length);
+}
+function computeOverall(areas) {
+    if (areas.length === 0) {
+        return { avgF1: 0, avgPrecision: 0, avgRecall: 0 };
+    }
+    // Weight by task count for fair averaging
+    const totalTasks = areas.reduce((s, a) => s + a.taskCount, 0);
+    if (totalTasks === 0) {
+        return { avgF1: 0, avgPrecision: 0, avgRecall: 0 };
+    }
+    const avgRecall = areas.reduce((s, a) => s + a.avgRecall * a.taskCount, 0) / totalTasks;
+    const avgPrecision = areas.reduce((s, a) => s + a.avgPrecision * a.taskCount, 0) / totalTasks;
+    const avgF1 = areas.reduce((s, a) => s + a.avgF1 * a.taskCount, 0) / totalTasks;
+    return { avgF1, avgPrecision, avgRecall };
+}
+function generateRecommendations(invisibleDocs, areas, overall) {
+    const recs = [];
+    // Recommend adding invisible docs to llms.txt
+    const highImpactInvisible = invisibleDocs.filter((d) => d.referencedBy.length > 0);
+    for (const doc of highImpactInvisible.slice(0, 5)) {
+        const taskWord = doc.referencedBy.length === 1 ? "task" : "tasks";
+        recs.push(`Add \`${doc.slug}\` to llms.txt (referenced by ${doc.referencedBy.length} ${taskWord})`);
+    }
+    // Recommend cross-linking for invisible docs
+    if (invisibleDocs.length > 0) {
+        recs.push(`Improve cross-linking to ${invisibleDocs.length} invisible document${invisibleDocs.length === 1 ? "" : "s"}`);
+    }
+    // Flag low-recall areas
+    const lowRecallAreas = areas.filter((a) => a.avgRecall < 0.5);
+    for (const area of lowRecallAreas) {
+        recs.push(`Investigate low recall in \`${area.area}\` (${pct(area.avgRecall)}) — agents miss most canonical docs`);
+    }
+    // Flag low-precision areas
+    const lowPrecisionAreas = areas.filter((a) => a.avgPrecision < 0.5);
+    for (const area of lowPrecisionAreas) {
+        recs.push(`Review search relevance for \`${area.area}\` (precision ${pct(area.avgPrecision)}) — agents fetch many irrelevant docs`);
+    }
+    // Overall recommendation
+    if (overall.avgF1 < 0.6) {
+        recs.push("Overall F1 is below 0.60 — consider a documentation restructure for agent accessibility");
+    }
+    return recs;
+}
+function pct(value) {
+    return `${Math.round(value * 100)}%`;
+}
+function sortedAreas(areas) {
+    return [...areas].sort((a, b) => a.area.localeCompare(b.area));
+}

package/dist/pipeline/eval-constants.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Constants and types shared across evaluation steps.
+ *
+ * Extracted from pipeline/steps/eval-step.ts so that the legacy step
+ * files can be deleted while tests and other modules retain access
+ * to these shared definitions.
+ */
+import type { ConcreteEvalMode, DebugOptions, FilterOptions, StepResult } from "../_vendor/ailf-core/index.d.ts";
+/** Promptfoo config file per concrete eval mode */
+export declare const CONFIG_FILES: Record<ConcreteEvalMode, string>;
+/** Each mode writes eval results to a different file (set in the config's outputPath) */
+export declare const RESULTS_FILES: Record<ConcreteEvalMode, string>;
+/** Extended step result that carries cache metadata for downstream steps */
+export interface EvalStepResult {
+    /** The computed eval fingerprint (for publishing in provenance) */
+    evalFingerprint?: string;
+    /** Whether this result came from a remote cache hit */
+    remoteCacheHit?: boolean;
+    /** The step result */
+    stepResult: StepResult;
+}
+/** Options for the remote cache (Content Lake fingerprint lookup) */
+export interface RemoteCacheOptions {
+    /** Whether this is a debug run (debug runs don't use remote cache) */
+    debug?: boolean;
+    /** Filter options used for fingerprint computation */
+    filter?: FilterOptions;
+    /** Grader model identifier from models.yaml */
+    graderModel: string;
+    /** Disable remote cache lookup (--no-remote-cache) */
+    noRemoteCache?: boolean;
+    /** Sanity API token for reading cached reports */
+    sanityToken?: string;
+}
+/** Minimal shape of a raw Promptfoo result entry for error scanning */
+export interface RawResult {
+    description?: string;
+    error?: string;
+    gradingResult?: null | {
+        pass: boolean;
+    };
+    provider?: {
+        id?: string;
+        label?: string;
+    };
+    success?: boolean;
+    testCase?: {
+        description?: string;
+    };
+}
+/**
+ * Build promptfoo filter flags from debug options.
+ */
+export declare function buildFilterFlags(debug?: DebugOptions): string;
+/**
+ * Extract the Promptfoo share URL from the eval results JSON.
+ *
+ * Promptfoo writes a `shareableUrl` field into the results file when
+ * `PROMPTFOO_API_KEY` is set.
+ */
+export declare function extractShareUrl(resultsPath: string): string | undefined;
+/**
+ * Read the eval results JSON and produce a human-readable summary of any
+ * errored or failed tests.
+ *
+ * Returns null if there are no errors/failures worth reporting.
+ */
+export declare function scanResultsForErrors(resultsPath: string): null | string;

package/dist/pipeline/eval-constants.js

@@ -0,0 +1,111 @@
+/**
+ * Constants and types shared across evaluation steps.
+ *
+ * Extracted from pipeline/steps/eval-step.ts so that the legacy step
+ * files can be deleted while tests and other modules retain access
+ * to these shared definitions.
+ */
+import { existsSync, readFileSync } from "fs";
+/** Promptfoo config file per concrete eval mode */
+export const CONFIG_FILES = {
+    agentic: "promptfooconfig.agentic.yaml",
+    baseline: "promptfooconfig.yaml",
+    observed: "promptfooconfig.observed.yaml",
+};
+/** Each mode writes eval results to a different file (set in the config's outputPath) */
+export const RESULTS_FILES = {
+    agentic: "results/latest/eval-results-agentic.json",
+    baseline: "results/latest/eval-results.json",
+    observed: "results/latest/eval-results-observed.json",
+};
+/**
+ * Build promptfoo filter flags from debug options.
+ */
+export function buildFilterFlags(debug) {
+    if (!debug?.enabled)
+        return "";
+    const flags = [];
+    if (debug.pattern) {
+        flags.push(`--filter-pattern '${debug.pattern}'`);
+    }
+    if (debug.sample) {
+        flags.push(`--filter-sample ${debug.sample}`);
+    }
+    if (debug.firstN) {
+        flags.push(`--filter-first-n ${debug.firstN}`);
+    }
+    // Default: first 2 tests when no other filters specified
+    if (flags.length === 0) {
+        flags.push("--filter-first-n 2");
+    }
+    return " " + flags.join(" ");
+}
+/**
+ * Extract the Promptfoo share URL from the eval results JSON.
+ *
+ * Promptfoo writes a `shareableUrl` field into the results file when
+ * `PROMPTFOO_API_KEY` is set.
+ */
+export function extractShareUrl(resultsPath) {
+    if (!existsSync(resultsPath))
+        return undefined;
+    try {
+        const raw = readFileSync(resultsPath, "utf-8");
+        const data = JSON.parse(raw);
+        return data.shareableUrl ?? undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Read the eval results JSON and produce a human-readable summary of any
+ * errored or failed tests.
+ *
+ * Returns null if there are no errors/failures worth reporting.
+ */
+export function scanResultsForErrors(resultsPath) {
+    if (!existsSync(resultsPath))
+        return null;
+    let file;
+    try {
+        const raw = readFileSync(resultsPath, "utf-8");
+        file = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    const results = file?.results?.results;
+    if (!Array.isArray(results))
+        return null;
+    const errored = [];
+    for (const r of results) {
+        if (r.gradingResult !== null)
+            continue;
+        const desc = r.testCase?.description ?? r.description ?? "unknown";
+        const provider = r.provider?.label ?? r.provider?.id ?? "unknown";
+        const errorMsg = r.error
+            ? (typeof r.error === "string" ? r.error : JSON.stringify(r.error)).slice(0, 200)
+            : "Provider returned no scorable result";
+        errored.push({ description: desc, error: errorMsg, provider });
+    }
+    if (errored.length === 0)
+        return null;
+    const total = results.length;
+    const lines = [];
+    lines.push(`  ┌─────────────────────────────────────────────────────────────`);
+    lines.push(`  │ ⚠️  ${errored.length} of ${total} eval result(s) errored (no gradingResult)`);
+    lines.push(`  │`);
+    for (const e of errored) {
+        lines.push(`  │   ✗ [${e.provider}] ${e.description}`);
+        lines.push(`  │     → ${e.error}`);
+    }
+    const errorRate = Math.round((errored.length / total) * 100);
+    if (errorRate >= 25) {
+        lines.push(`  │`);
+        lines.push(`  │ 🔥 High error rate (${errorRate}%) — check API keys, rate limits,`);
+        lines.push(`  │   or model availability. Errored results are excluded from scoring.`);
+    }
+    lines.push(`  └─────────────────────────────────────────────────────────────`);
+    return lines.join("\n");
+}