@sanity/ailf 0.1.0

package/dist/pipeline/grader-validate-runner.js

@@ -0,0 +1,229 @@
+/**
+ * pipeline/grader-validate-runner.ts
+ *
+ * Orchestration module for grader validation against human reference grades
+ * (Phase 2).
+ *
+ * Loads human-graded reference samples from canonical/grader-references/,
+ * runs the grader model on each sample, and compares against human scores
+ * using `validateGrader()` from the pure computation module.
+ *
+ * Migrated from lib/grader-validate.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 2
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "fs";
+import { join } from "path";
+import { load } from "js-yaml";
+import { gradeOnce, loadGraderModel } from "./grader-api.js";
+import { classifyCorrelation, validateGrader, } from "./grader-validation.js";
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function mapDimension(dim) {
+    switch (dim) {
+        case "code-correctness":
+            return "codeCorrectness";
+        case "doc-coverage":
+            return "docCoverage";
+        case "task-completion":
+            return "taskCompletion";
+        default:
+            return null;
+    }
+}
+/**
+ * Load human reference grades from canonical/grader-references/.
+ * @throws Error if directory or files not found
+ */
+function loadReferenceGrades(rootDir) {
+    const refsDir = join(rootDir, "canonical", "grader-references");
+    if (!existsSync(refsDir)) {
+        throw new Error(`Reference grades directory not found: ${refsDir}. ` +
+            "Create canonical/grader-references/ with YAML reference files. " +
+            "See docs/exec-plans/completed/grader-reliability.md — Phase 2.");
+    }
+    const files = readdirSync(refsDir)
+        .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+        .sort();
+    if (files.length === 0) {
+        throw new Error(`No YAML files found in ${refsDir}`);
+    }
+    const allGrades = [];
+    for (const file of files) {
+        const filePath = join(refsDir, file);
+        const raw = readFileSync(filePath, "utf-8");
+        const parsed = load(raw);
+        if (Array.isArray(parsed)) {
+            allGrades.push(...parsed);
+        }
+        else if (typeof parsed === "object" && parsed !== null) {
+            allGrades.push(parsed);
+        }
+    }
+    return allGrades;
+}
+// ---------------------------------------------------------------------------
+// Report formatting (pure)
+// ---------------------------------------------------------------------------
+/**
+ * Format a GraderValidation result as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export function formatValidationReport(result) {
+    const lines = [];
+    lines.push("=".repeat(80));
+    lines.push("                    GRADER VALIDATION REPORT");
+    lines.push("=".repeat(80));
+    lines.push("");
+    lines.push(`  Grader:          ${result.graderModel}`);
+    lines.push(`  Observations:    ${result.totalObservations}`);
+    lines.push("");
+    // Overall metrics
+    lines.push("-".repeat(80));
+    lines.push("OVERALL METRICS");
+    lines.push("-".repeat(80));
+    lines.push("");
+    lines.push(`  MAE:             ${result.overallMae} points`);
+    lines.push(`  Correlation:     r=${result.overallCorrelation} (${classifyCorrelation(result.overallCorrelation)})`);
+    lines.push(`  Bias:            ${result.overallBias > 0 ? "+" : ""}${result.overallBias} (${result.overallBias > 0 ? "grader scores higher" : result.overallBias < 0 ? "grader scores lower" : "no bias"})`);
+    lines.push("");
+    // Per-dimension table
+    lines.push("-".repeat(80));
+    lines.push("PER-DIMENSION VALIDITY");
+    lines.push("-".repeat(80));
+    lines.push("");
+    const h = "| Dimension        |  MAE  | Correlation | Quality   | Bias   | Count |";
+    const sep = "|------------------|-------|-------------|-----------|--------|-------|";
+    lines.push(h);
+    lines.push(sep);
+    const dims = [
+        { data: result.perDimension.taskCompletion, name: "Task Completion" },
+        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
+        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
+    ];
+    for (const { data, name } of dims) {
+        const quality = classifyCorrelation(data.correlation);
+        const biasStr = data.bias > 0 ? `+${data.bias}` : `${data.bias}`;
+        lines.push(`| ${name.padEnd(16)} | ${String(data.mae).padStart(5)} | r=${String(data.correlation).padStart(9)} | ${quality.padEnd(9)} | ${biasStr.padStart(6)} | ${String(data.count).padStart(5)} |`);
+    }
+    lines.push("");
+    // Pass/fail verdict
+    lines.push("-".repeat(80));
+    lines.push("VERDICT");
+    lines.push("-".repeat(80));
+    lines.push("");
+    if (result.passesThreshold) {
+        lines.push(`  ✅ PASSED: MAE ${result.overallMae} < threshold ${result.maeThreshold}`);
+    }
+    else {
+        lines.push(`  ❌ FAILED: MAE ${result.overallMae} >= threshold ${result.maeThreshold}`);
+    }
+    lines.push("");
+    // Largest disagreements
+    const topN = Math.min(5, result.largestDisagreements.length);
+    if (topN > 0) {
+        lines.push("-".repeat(80));
+        lines.push(`TOP ${topN} LARGEST DISAGREEMENTS`);
+        lines.push("-".repeat(80));
+        lines.push("");
+        for (let i = 0; i < topN; i++) {
+            const d = result.largestDisagreements[i];
+            const sign = d.signedError > 0 ? "+" : "";
+            lines.push(`  ${i + 1}. ${d.taskId} — ${d.dimension}`);
+            lines.push(`     Human=${d.humanScore}, Grader=${d.graderScore} (${sign}${d.signedError})`);
+            if (d.notes) {
+                lines.push(`     Note: ${d.notes}`);
+            }
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+/**
+ * Run grader validation against human reference grades.
+ *
+ * Loads human-graded samples, grades each with the grader model,
+ * and computes validation metrics (MAE, correlation, bias).
+ *
+ * @throws Error if no reference grades found, or no grades to analyze
+ */
+export async function runGraderValidate(options) {
+    const { rootDir } = options;
+    const maeThreshold = options.maeThreshold ?? 10;
+    console.log("=== Grader Validation ===\n");
+    // Resolve grader model
+    const graderModel = options.graderModel ?? loadGraderModel(rootDir).id;
+    console.log(`  Grader:     ${graderModel}`);
+    console.log(`  Threshold:  MAE < ${maeThreshold}`);
+    // Load reference grades
+    const rawGrades = loadReferenceGrades(rootDir);
+    console.log(`  Samples:    ${rawGrades.length} reference-graded responses`);
+    // Count total rubric judgments
+    let totalJudgments = 0;
+    for (const rg of rawGrades) {
+        totalJudgments += rg.rubrics.length;
+    }
+    console.log(`  Judgments:  ${totalJudgments} (response × rubric pairs)`);
+    const estimatedCost = totalJudgments * 0.005;
+    console.log(`  Est. cost:  ~$${estimatedCost.toFixed(2)}`);
+    console.log();
+    // Grade each reference sample
+    console.log("  Running grader on reference samples...");
+    const grades = [];
+    let completed = 0;
+    let failed = 0;
+    for (const ref of rawGrades) {
+        for (const rubric of ref.rubrics) {
+            const dimension = mapDimension(rubric.dimension);
+            if (!dimension) {
+                console.error(`  ⚠ Unknown dimension '${rubric.dimension}' — skipping`);
+                continue;
+            }
+            const graderScore = await gradeOnce(graderModel, ref.response, rubric.rubricText);
+            completed++;
+            if (completed % 5 === 0 || completed === totalJudgments) {
+                process.stdout.write(`\r  Progress: ${completed}/${totalJudgments}`);
+            }
+            if (graderScore === null) {
+                failed++;
+                continue;
+            }
+            grades.push({
+                area: ref.area,
+                dimension,
+                graderScore,
+                humanScore: rubric.humanScore,
+                taskId: ref.taskId,
+                ...(rubric.notes && { notes: rubric.notes }),
+            });
+        }
+    }
+    console.log(); // newline after progress
+    if (failed > 0) {
+        console.log(`  ⚠ ${failed} grading calls failed (excluded from analysis)`);
+    }
+    console.log();
+    if (grades.length === 0) {
+        throw new Error("No grades to analyze. All grading calls failed.");
+    }
+    // Validate
+    const result = validateGrader(grades, graderModel, { maeThreshold });
+    // Print report
+    console.log(formatValidationReport(result));
+    // Write output
+    const outDir = join(rootDir, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    const outPath = join(outDir, "grader-validation.json");
+    writeFileSync(outPath, JSON.stringify(result, null, 2));
+    console.log(`\n  📄 Results written to ${outPath}`);
+    // Throw if threshold not met (instead of process.exit)
+    if (!result.passesThreshold) {
+        throw new Error(`VALIDATION FAILED: MAE ${result.overallMae} exceeds threshold ${maeThreshold}`);
+    }
+    return result;
+}

package/dist/pipeline/grader-validation.d.ts

@@ -0,0 +1,107 @@
+/**
+ * pipeline/grader-validation.ts
+ *
+ * Pure computation module for grader validation against human reference grades.
+ *
+ * Takes paired (graderScore, humanScore) observations and computes:
+ * - Mean Absolute Error (MAE)
+ * - Pearson correlation
+ * - Per-dimension breakdown
+ * - Largest disagreements (for diagnosis)
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 2
+ */
+/** Quality label for a correlation value */
+export type CorrelationQuality = "excellent" | "good" | "moderate" | "poor" | "very-poor";
+/** Per-dimension validity metrics */
+export interface DimensionValidity {
+    /** Mean signed error (positive = grader scores higher than human) */
+    bias: number;
+    /** Pearson correlation coefficient (-1 to 1) */
+    correlation: number;
+    /** Number of observations */
+    count: number;
+    /** Mean Absolute Error for this dimension */
+    mae: number;
+}
+/** A single large disagreement between grader and human */
+export interface Disagreement {
+    /** Absolute difference */
+    absoluteError: number;
+    /** Feature area */
+    area: string;
+    /** Scoring dimension */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Grader score */
+    graderScore: number;
+    /** Human score */
+    humanScore: number;
+    /** Optional notes from the human grader */
+    notes?: string;
+    /** Signed difference (grader - human) */
+    signedError: number;
+    /** Task ID */
+    taskId: string;
+}
+/** Top-level grader validation result */
+export interface GraderValidation {
+    /** When this validation was generated */
+    generatedAt: string;
+    /** Grader model that was validated */
+    graderModel: string;
+    /** Top N largest disagreements (sorted by absolute error, descending) */
+    largestDisagreements: Disagreement[];
+    /** The MAE threshold used */
+    maeThreshold: number;
+    /** Overall bias (positive = grader grades higher than humans on average) */
+    overallBias: number;
+    /** Overall Pearson correlation */
+    overallCorrelation: number;
+    /** Overall Mean Absolute Error across all dimensions */
+    overallMae: number;
+    /** Whether the grader passes the MAE threshold (default: MAE < 10) */
+    passesThreshold: boolean;
+    /** Per-dimension validity metrics */
+    perDimension: {
+        taskCompletion: DimensionValidity;
+        codeCorrectness: DimensionValidity;
+        docCoverage: DimensionValidity;
+    };
+    /** Total number of (grader, human) score pairs analyzed */
+    totalObservations: number;
+}
+/** A single human reference grade for one (response, rubric) judgment */
+export interface HumanReferenceGrade {
+    /** Feature area */
+    area: string;
+    /** Which scoring dimension */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** The grader's score (0–100) — populated after grading */
+    graderScore: number;
+    /** The human expert's score (0–100) */
+    humanScore: number;
+    /** Optional notes from the human grader */
+    notes?: string;
+    /** Which task this judgment belongs to */
+    taskId: string;
+}
+/** Classify a correlation value into a human-readable quality label */
+export declare function classifyCorrelation(r: number): CorrelationQuality;
+/** Compute Pearson correlation coefficient between two arrays of equal length */
+export declare function pearsonCorrelation(x: number[], y: number[]): number;
+/**
+ * Validate grader scores against human reference grades.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param grades Array of paired (grader, human) observations
+ * @param graderModel The grader model identifier
+ * @param options Optional configuration
+ * @returns GraderValidation report
+ */
+export declare function validateGrader(grades: HumanReferenceGrade[], graderModel: string, options?: {
+    maeThreshold?: number;
+    maxDisagreements?: number;
+}): GraderValidation;

package/dist/pipeline/grader-validation.js

@@ -0,0 +1,169 @@
+/**
+ * pipeline/grader-validation.ts
+ *
+ * Pure computation module for grader validation against human reference grades.
+ *
+ * Takes paired (graderScore, humanScore) observations and computes:
+ * - Mean Absolute Error (MAE)
+ * - Pearson correlation
+ * - Per-dimension breakdown
+ * - Largest disagreements (for diagnosis)
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 2
+ */
+// ---------------------------------------------------------------------------
+// Pure computation
+// ---------------------------------------------------------------------------
+/** Classify a correlation value into a human-readable quality label */
+export function classifyCorrelation(r) {
+    const abs = Math.abs(r);
+    if (abs >= 0.9)
+        return "excellent";
+    if (abs >= 0.7)
+        return "good";
+    if (abs >= 0.5)
+        return "moderate";
+    if (abs >= 0.3)
+        return "poor";
+    return "very-poor";
+}
+/** Compute Pearson correlation coefficient between two arrays of equal length */
+export function pearsonCorrelation(x, y) {
+    const n = x.length;
+    if (n < 2)
+        return 0;
+    if (x.length !== y.length) {
+        throw new Error(`Arrays must have equal length (got ${x.length} and ${y.length})`);
+    }
+    const meanX = x.reduce((s, v) => s + v, 0) / n;
+    const meanY = y.reduce((s, v) => s + v, 0) / n;
+    let sumXY = 0;
+    let sumX2 = 0;
+    let sumY2 = 0;
+    for (let i = 0; i < n; i++) {
+        const dx = x[i] - meanX;
+        const dy = y[i] - meanY;
+        sumXY += dx * dy;
+        sumX2 += dx * dx;
+        sumY2 += dy * dy;
+    }
+    const denom = Math.sqrt(sumX2 * sumY2);
+    if (denom === 0)
+        return 0;
+    return sumXY / denom;
+}
+/**
+ * Validate grader scores against human reference grades.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param grades Array of paired (grader, human) observations
+ * @param graderModel The grader model identifier
+ * @param options Optional configuration
+ * @returns GraderValidation report
+ */
+export function validateGrader(grades, graderModel, options) {
+    const maeThreshold = options?.maeThreshold ?? 10;
+    const maxDisagreements = options?.maxDisagreements ?? 10;
+    if (grades.length === 0) {
+        return {
+            generatedAt: new Date().toISOString(),
+            graderModel,
+            largestDisagreements: [],
+            maeThreshold,
+            overallBias: 0,
+            overallCorrelation: 0,
+            overallMae: 0,
+            passesThreshold: true,
+            perDimension: {
+                codeCorrectness: { bias: 0, correlation: 0, count: 0, mae: 0 },
+                docCoverage: { bias: 0, correlation: 0, count: 0, mae: 0 },
+                taskCompletion: { bias: 0, correlation: 0, count: 0, mae: 0 },
+            },
+            totalObservations: 0,
+        };
+    }
+    // Build paired arrays
+    const allPairs = grades.map((g) => ({
+        grader: g.graderScore,
+        human: g.humanScore,
+    }));
+    // Group by dimension
+    const byDimension = {
+        codeCorrectness: grades
+            .filter((g) => g.dimension === "codeCorrectness")
+            .map((g) => ({ grader: g.graderScore, human: g.humanScore })),
+        docCoverage: grades
+            .filter((g) => g.dimension === "docCoverage")
+            .map((g) => ({ grader: g.graderScore, human: g.humanScore })),
+        taskCompletion: grades
+            .filter((g) => g.dimension === "taskCompletion")
+            .map((g) => ({ grader: g.graderScore, human: g.humanScore })),
+    };
+    // Overall metrics
+    const overallMae = computeMae(allPairs);
+    const overallCorrelation = Math.round(pearsonCorrelation(allPairs.map((p) => p.grader), allPairs.map((p) => p.human)) * 100) / 100;
+    const overallBias = computeBias(allPairs);
+    // Per-dimension metrics
+    const perDimension = {
+        codeCorrectness: computeDimensionValidity(byDimension.codeCorrectness),
+        docCoverage: computeDimensionValidity(byDimension.docCoverage),
+        taskCompletion: computeDimensionValidity(byDimension.taskCompletion),
+    };
+    // Find largest disagreements
+    const disagreements = grades
+        .map((g) => ({
+        absoluteError: Math.abs(g.graderScore - g.humanScore),
+        area: g.area,
+        dimension: g.dimension,
+        graderScore: g.graderScore,
+        humanScore: g.humanScore,
+        signedError: g.graderScore - g.humanScore,
+        taskId: g.taskId,
+        ...(g.notes && { notes: g.notes }),
+    }))
+        .sort((a, b) => b.absoluteError - a.absoluteError)
+        .slice(0, maxDisagreements);
+    return {
+        generatedAt: new Date().toISOString(),
+        graderModel,
+        largestDisagreements: disagreements,
+        maeThreshold,
+        overallBias,
+        overallCorrelation,
+        overallMae,
+        passesThreshold: overallMae < maeThreshold,
+        perDimension,
+        totalObservations: grades.length,
+    };
+}
+/** Compute mean signed error (bias) */
+function computeBias(pairs) {
+    if (pairs.length === 0)
+        return 0;
+    const totalBias = pairs.reduce((sum, p) => sum + (p.grader - p.human), 0);
+    return Math.round((totalBias / pairs.length) * 10) / 10;
+}
+/** Compute validity metrics for a set of paired observations */
+function computeDimensionValidity(pairs) {
+    if (pairs.length === 0) {
+        return { bias: 0, correlation: 0, count: 0, mae: 0 };
+    }
+    const graderScores = pairs.map((p) => p.grader);
+    const humanScores = pairs.map((p) => p.human);
+    return {
+        bias: computeBias(pairs),
+        correlation: Math.round(pearsonCorrelation(graderScores, humanScores) * 100) / 100,
+        count: pairs.length,
+        mae: computeMae(pairs),
+    };
+}
+/** Compute Mean Absolute Error */
+function computeMae(pairs) {
+    if (pairs.length === 0)
+        return 0;
+    const totalError = pairs.reduce((sum, p) => sum + Math.abs(p.grader - p.human), 0);
+    return Math.round((totalError / pairs.length) * 10) / 10;
+}

package/dist/pipeline/map-request-to-config.d.ts

@@ -0,0 +1,19 @@
+import type { PipelineRequest, ResolvedConfig } from "../_vendor/ailf-core/index.d.ts";
+/**
+ * Map a PipelineRequest to a ResolvedConfig.
+ *
+ * This is the single mapping function shared between:
+ * - API gateway (POST /v1/pipeline → PipelineRequest → ResolvedConfig)
+ * - CLI --config (JSON file → PipelineRequest → ResolvedConfig)
+ * - GitHub Actions (repository_dispatch payload → PipelineRequest → ResolvedConfig)
+ *
+ * Fields not present in PipelineRequest get sensible defaults.
+ * Fields that only exist in ResolvedConfig (rootDir, promptfooUrl, etc.)
+ * must be provided by the caller via the rootDir parameter.
+ *
+ * **Publish default:** When `jobId` is present (API-triggered evaluation),
+ * `publish` defaults to `true` so the report is persisted to the Content
+ * Lake and the job document gets a `reportId`. Callers can still override
+ * with `publish: false`.
+ */
+export declare function mapRequestToConfig(request: PipelineRequest, rootDir: string): ResolvedConfig;

package/dist/pipeline/map-request-to-config.js

@@ -0,0 +1,80 @@
+/**
+ * Map a PipelineRequest to a ResolvedConfig.
+ *
+ * This is the single mapping function shared between:
+ * - API gateway (POST /v1/pipeline → PipelineRequest → ResolvedConfig)
+ * - CLI --config (JSON file → PipelineRequest → ResolvedConfig)
+ * - GitHub Actions (repository_dispatch payload → PipelineRequest → ResolvedConfig)
+ *
+ * Fields not present in PipelineRequest get sensible defaults.
+ * Fields that only exist in ResolvedConfig (rootDir, promptfooUrl, etc.)
+ * must be provided by the caller via the rootDir parameter.
+ *
+ * **Publish default:** When `jobId` is present (API-triggered evaluation),
+ * `publish` defaults to `true` so the report is persisted to the Content
+ * Lake and the job document gets a `reportId`. Callers can still override
+ * with `publish: false`.
+ */
+export function mapRequestToConfig(request, rootDir) {
+    // API-triggered evaluations (identified by jobId) default to publish: true.
+    // Without this, the job's reportId is always null and GET /v1/reports/:id
+    // has nothing to return.
+    const publishDefault = !!request.jobId;
+    return {
+        rootDir,
+        mode: request.mode ?? "full",
+        debug: mapDebug(request.debug),
+        areas: request.areas,
+        tasks: request.tasks,
+        changedDocs: request.changedDocs,
+        source: request.source,
+        skipFetch: false,
+        skipEval: false,
+        compareEnabled: request.compare ?? false,
+        compareThreshold: request.compareThreshold,
+        compareBaseline: request.compareBaseline,
+        gapAnalysisEnabled: request.gapAnalysis ?? true,
+        readinessEnabled: request.readiness ?? false,
+        discoveryReportEnabled: request.discoveryReport ?? false,
+        publishEnabled: request.publish ?? publishDefault,
+        publishTag: request.publishTag,
+        noCache: request.noCache ?? false,
+        noRemoteCache: request.noRemoteCache ?? false,
+        graderReplications: request.graderReplications,
+        urls: request.urls,
+        headers: request.headers,
+        allowedOrigins: request.allowedOrigins,
+        searchMode: request.searchMode ?? "open",
+        concurrency: request.concurrency,
+        datasetOverride: request.dataset,
+        projectIdOverride: request.projectId,
+        perspectiveOverride: request.perspective,
+        taskSourceType: mapTaskSourceType(request.taskMode),
+        outputPath: undefined,
+        promptfooUrl: undefined,
+        studioOriginOverride: undefined,
+        sanityDocumentArgs: undefined,
+        beforeOption: undefined,
+        repoTasksPath: undefined,
+        callback: request.callback,
+        jobId: request.jobId,
+    };
+}
+function mapDebug(debug) {
+    if (debug === undefined || debug === false)
+        return undefined;
+    if (debug === true)
+        return { enabled: true };
+    return {
+        enabled: debug.enabled ?? true,
+        firstN: debug.firstN,
+        pattern: debug.pattern,
+        sample: debug.sample,
+    };
+}
+function mapTaskSourceType(taskMode) {
+    if (taskMode === "content-lake" || taskMode === "yaml")
+        return taskMode;
+    // "inline" is handled separately by the caller; maps to undefined
+    return undefined;
+}

package/dist/pipeline/measure-retrieval.d.ts

@@ -0,0 +1,59 @@
+/**
+ * measure-retrieval.ts
+ *
+ * Pure retrieval quality measurement functions.
+ *
+ * Evaluates retrieval quality by comparing what Sanity's text search
+ * returns against the manually-annotated canonical documents for each
+ * evaluation task. Produces Recall@K and NDCG@K metrics.
+ *
+ * This answers: "Can a retriever find the docs an LLM actually needs?"
+ *
+ * Migrated from lib/measure-retrieval.ts — no process.argv/process.env,
+ * accepts rootDir and retriever function as parameters.
+ */
+export interface RetrievalResult {
+    canonical_docs: string[];
+    feature_area: string;
+    ndcg_at_10: number;
+    recall_at_5: number;
+    recall_at_10: number;
+    retrieved_docs: string[];
+    task_id: string;
+}
+export interface RetrievalSummary {
+    by_area: Record<string, {
+        avg_recall_at_5: number;
+        avg_recall_at_10: number;
+        avg_ndcg_at_10: number;
+        task_count: number;
+    }>;
+    overall: {
+        avg_recall_at_5: number;
+        avg_recall_at_10: number;
+        avg_ndcg_at_10: number;
+    };
+    results: RetrievalResult[];
+}
+/** A retriever function: given a query and k, returns k document slugs. */
+export type RetrieverFn = (query: string, k: number) => Promise<string[]>;
+export declare function calculateRecall(canonical: string[], retrieved: string[], k: number): number;
+export declare function calculateNDCG(canonical: string[], retrieved: string[], k: number): number;
+export interface MeasureRetrievalOptions {
+    /** Root directory of the eval package */
+    rootDir: string;
+    /** Retriever function — injected by callers for testability */
+    retriever: RetrieverFn;
+    /** Optional progress callback */
+    onProgress?: (area: string, taskId: string, result: RetrievalResult) => void;
+}
+/**
+ * Run retrieval quality measurement for all tasks.
+ *
+ * @returns A full RetrievalSummary with per-task, per-area, and overall metrics.
+ */
+export declare function measureRetrieval(options: MeasureRetrievalOptions): Promise<RetrievalSummary>;
+/**
+ * Format a retrieval summary for console display.
+ */
+export declare function formatRetrievalTable(summary: RetrievalSummary): string;