@sanity/ailf 0.1.0

package/dist/pipeline/grader-consistency-runner.js

@@ -0,0 +1,270 @@
+/**
+ * grader-consistency-runner.ts
+ *
+ * Orchestration logic for grader consistency analysis.
+ *
+ * Reads existing eval results, re-runs ONLY the grading assertions N additional
+ * times with the configured grader model, and analyzes score variance.
+ *
+ * This does NOT re-run the models under test — it only re-grades the same
+ * responses. Cost is low: ~$0.005 per grading call × N replications.
+ *
+ * Migrated from lib/grader-consistency.ts — no process.argv, no process.exit(),
+ * no module-level constants.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 1
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { gradeOnce } from "./grader-api.js";
+import { analyzeConsistency, } from "./grader-consistency.js";
+// ---------------------------------------------------------------------------
+// Rubric dimension classification (similar to calculate-scores)
+// ---------------------------------------------------------------------------
+function classifyDimension(component) {
+    // Prefer structured metadata
+    const metadata = component.assertion?.metadata;
+    if (metadata?.dimension) {
+        switch (metadata.dimension) {
+            case "code-correctness":
+                return "codeCorrectness";
+            case "doc-coverage":
+                return "docCoverage";
+            case "task-completion":
+                return "taskCompletion";
+            default:
+                return null;
+        }
+    }
+    // Fallback: heuristic name matching
+    const value = (component.assertion?.value ?? "").toLowerCase();
+    if (value.includes("task completion"))
+        return "taskCompletion";
+    if (value.includes("code correctness"))
+        return "codeCorrectness";
+    if (value.includes("documentation coverage") || value.includes("hallucinate"))
+        return "docCoverage";
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Grading judgment extraction
+// ---------------------------------------------------------------------------
+function detectFeatureArea(description) {
+    const desc = description.toLowerCase();
+    if (desc.includes("studio"))
+        return "studio-setup";
+    if (desc.includes("visual") ||
+        desc.includes("presentation") ||
+        desc.includes("live preview"))
+        return "visual-editing";
+    if (desc.includes("function") || desc.includes("webhook"))
+        return "functions";
+    if (desc.startsWith("groq"))
+        return "groq";
+    if (desc.includes("next") || desc.includes("app router"))
+        return "nextjs-live";
+    if (desc.includes("remix") ||
+        desc.includes("nuxt") ||
+        desc.includes("svelte"))
+        return "frameworks";
+    return "other";
+}
+/**
+ * Extract all llm-rubric grading judgments from eval results.
+ * Only includes gold (with-docs) tests to keep the analysis focused.
+ */
+export function extractGradingJudgments(file) {
+    const judgments = [];
+    for (const result of file.results.results) {
+        if (!result.gradingResult)
+            continue;
+        const description = result.testCase?.description ?? "unknown";
+        const hasDocs = result.vars?.docs && result.vars.docs.trim().length > 0;
+        // Only grade "gold" (with-docs) tests — baseline tests have abbreviated rubrics
+        if (!hasDocs)
+            continue;
+        const area = detectFeatureArea(description);
+        const providerId = result.provider?.id;
+        for (const comp of result.gradingResult.componentResults) {
+            if (comp.assertion?.type !== "llm-rubric")
+                continue;
+            const dimension = classifyDimension(comp);
+            if (!dimension)
+                continue;
+            const rubricText = typeof comp.assertion.value === "string" ? comp.assertion.value : "";
+            if (!rubricText)
+                continue;
+            judgments.push({
+                area,
+                description,
+                dimension,
+                originalScore: typeof comp.score === "number" ? comp.score : 0,
+                providerId,
+                responseText: result.response?.output ?? "",
+                rubricText,
+            });
+        }
+    }
+    return judgments;
+}
+// ---------------------------------------------------------------------------
+// Report formatting (pure)
+// ---------------------------------------------------------------------------
+export function formatConsistencyReport(result, graderModel) {
+    const lines = [];
+    lines.push("=".repeat(80));
+    lines.push("                    GRADER CONSISTENCY REPORT");
+    lines.push("=".repeat(80));
+    lines.push("");
+    lines.push(`  Grader model:  ${graderModel}`);
+    lines.push(`  Replications:  ${result.replications} (incl. original)`);
+    lines.push(`  Judgments:     ${result.totalJudgments}`);
+    lines.push("");
+    // Overall stats
+    lines.push("-".repeat(80));
+    lines.push("OVERALL");
+    lines.push("-".repeat(80));
+    lines.push("");
+    lines.push(`  Avg σ:          ${result.avgStdDev}`);
+    lines.push(`  Max σ:          ${result.maxStdDev}`);
+    lines.push(`  Avg range:      ${result.avgRange} points`);
+    lines.push("");
+    // Per-dimension table
+    lines.push("-".repeat(80));
+    lines.push("PER-DIMENSION CONSISTENCY");
+    lines.push("-".repeat(80));
+    lines.push("");
+    const h = "| Dimension        | Avg σ | Max σ | Avg Range | Judgments |";
+    const sep = "|------------------|-------|-------|-----------|-----------| ";
+    lines.push(h);
+    lines.push(sep);
+    const dims = [
+        { data: result.perDimension.taskCompletion, name: "Task Completion" },
+        { data: result.perDimension.codeCorrectness, name: "Code Correctness" },
+        { data: result.perDimension.docCoverage, name: "Doc Coverage" },
+    ];
+    for (const { data, name } of dims) {
+        lines.push(`| ${name.padEnd(16)} | ${String(data.avgStdDev).padStart(5)} | ${String(data.maxStdDev).padStart(5)} | ${String(data.avgRange).padStart(9)} | ${String(data.judgmentCount).padStart(9)} |`);
+    }
+    lines.push("");
+    // Noise threshold recommendation
+    lines.push("-".repeat(80));
+    lines.push("NOISE THRESHOLD RECOMMENDATION");
+    lines.push("-".repeat(80));
+    lines.push("");
+    lines.push(`  Current default:     ±2 (DEFAULT_NOISE_THRESHOLD)`);
+    lines.push(`  Recommended:         ±${result.recommendedThreshold} (based on 2× max dimension σ)`);
+    if (result.recommendedThreshold > 2) {
+        lines.push(`  ⚠ Current threshold may be too low — comparison deltas within ±${result.recommendedThreshold}`);
+        lines.push(`    should be treated as noise, not real changes.`);
+    }
+    else {
+        lines.push(`  ✅ Current threshold is adequate for this grader's consistency.`);
+    }
+    lines.push("");
+    // Top 5 noisiest judgments
+    const topN = Math.min(5, result.judgments.length);
+    if (topN > 0) {
+        lines.push("-".repeat(80));
+        lines.push(`TOP ${topN} NOISIEST JUDGMENTS`);
+        lines.push("-".repeat(80));
+        lines.push("");
+        for (let i = 0; i < topN; i++) {
+            const j = result.judgments[i];
+            const provider = j.providerId ? ` [${j.providerId}]` : "";
+            lines.push(`  ${i + 1}. ${j.taskId}${provider}`);
+            lines.push(`     ${j.dimension}: σ=${j.stdDev}, range=${j.range} (${j.min}–${j.max}), mean=${j.mean}`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Main runner
+// ---------------------------------------------------------------------------
+/**
+ * Run the grader consistency analysis.
+ *
+ * Reads eval results, re-grades each judgment N times, and writes the
+ * consistency report to results/latest/grader-consistency.json.
+ *
+ * @throws Error if results file not found, replications < 2, or no judgments found
+ */
+export async function runGraderConsistency(options) {
+    const { replications, resultsPath, rootDir } = options;
+    console.log("=== Grader Consistency Analysis ===\n");
+    // Validate inputs
+    if (!existsSync(resultsPath)) {
+        throw new Error(`Results file not found: ${resultsPath}. Run 'pnpm eval' first to generate results.`);
+    }
+    if (replications < 2) {
+        throw new Error("Need at least 2 replications for meaningful analysis.");
+    }
+    // Load eval results
+    console.log(`  Results:      ${resultsPath}`);
+    console.log(`  Replications: ${replications}`);
+    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    // Extract grader model
+    const graderModel = file.config?.defaultTest?.options?.rubricProvider ??
+        file.config?.defaultTest?.options?.provider;
+    if (!graderModel) {
+        throw new Error("Could not determine grader model from eval results config.");
+    }
+    console.log(`  Grader:       ${graderModel}`);
+    // Extract judgments
+    const judgments = extractGradingJudgments(file);
+    console.log(`  Judgments:    ${judgments.length} (gold tests × rubric dimensions)`);
+    if (judgments.length === 0) {
+        throw new Error("No gradable judgments found in results.");
+    }
+    const totalCalls = judgments.length * replications;
+    const estimatedCost = totalCalls * 0.005;
+    console.log(`  API calls:    ${totalCalls} (${judgments.length} × ${replications})`);
+    console.log(`  Est. cost:    ~$${estimatedCost.toFixed(2)}`);
+    console.log();
+    // Re-grade each judgment N times
+    console.log(`  Running ${replications} replications per judgment...`);
+    const gradings = [];
+    let completed = 0;
+    let failed = 0;
+    for (const judgment of judgments) {
+        const scores = [judgment.originalScore]; // Include the original as replication #0
+        for (let i = 0; i < replications; i++) {
+            const score = await gradeOnce(graderModel, judgment.responseText, judgment.rubricText);
+            if (score !== null) {
+                scores.push(score);
+            }
+            else {
+                failed++;
+            }
+        }
+        completed++;
+        if (completed % 10 === 0 || completed === judgments.length) {
+            const pct = Math.round((completed / judgments.length) * 100);
+            process.stdout.write(`\r  Progress: ${completed}/${judgments.length} (${pct}%)`);
+        }
+        gradings.push({
+            area: judgment.area,
+            dimension: judgment.dimension,
+            providerId: judgment.providerId,
+            scores,
+            taskId: judgment.description,
+        });
+    }
+    console.log(); // newline after progress
+    if (failed > 0) {
+        console.log(`  ⚠ ${failed} grading calls failed (excluded from analysis)`);
+    }
+    console.log();
+    // Analyze consistency
+    const result = analyzeConsistency(gradings);
+    // Print report
+    console.log(formatConsistencyReport(result, graderModel));
+    // Write output
+    const outDir = join(rootDir, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    const outPath = join(outDir, "grader-consistency.json");
+    writeFileSync(outPath, JSON.stringify(result, null, 2));
+    console.log(`\n  📄 Results written to ${outPath}`);
+    return result;
+}

package/dist/pipeline/grader-consistency.d.ts

@@ -0,0 +1,103 @@
+/**
+ * pipeline/grader-consistency.ts
+ *
+ * Pure computation module for measuring grader consistency.
+ *
+ * Takes replicated grading results (the same judgment run N times) and
+ * computes standard deviation, per-dimension consistency, and an
+ * empirically-derived noise threshold recommendation.
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ * It operates on pre-collected data only.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 1
+ */
+/** Per-dimension consistency aggregates */
+export interface DimensionConsistency {
+    /** Average range (max - min) across all judgments */
+    avgRange: number;
+    /** Average standard deviation across all judgments in this dimension */
+    avgStdDev: number;
+    /** Number of judgments contributing to this aggregate */
+    judgmentCount: number;
+    /** Maximum standard deviation observed in this dimension */
+    maxStdDev: number;
+}
+/** Top-level grader consistency result */
+export interface GraderConsistency {
+    /** Average range (max - min) across all judgments */
+    avgRange: number;
+    /** Average standard deviation across ALL grading judgments */
+    avgStdDev: number;
+    /** When this analysis was generated */
+    generatedAt: string;
+    /** Per-judgment details (sorted by stdDev descending — noisiest first) */
+    judgments: JudgmentConsistency[];
+    /** Maximum standard deviation observed (worst-case noise) */
+    maxStdDev: number;
+    /** Per-dimension consistency */
+    perDimension: {
+        taskCompletion: DimensionConsistency;
+        codeCorrectness: DimensionConsistency;
+        docCoverage: DimensionConsistency;
+    };
+    /** Recommended noise threshold for comparisons (2× max dimension avgStdDev) */
+    recommendedThreshold: number;
+    /** Number of replications per judgment */
+    replications: number;
+    /** Total number of judgments analyzed */
+    totalJudgments: number;
+}
+/** Consistency stats for a single judgment */
+export interface JudgmentConsistency {
+    /** Feature area */
+    area: string;
+    /** Scoring dimension */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Max score observed */
+    max: number;
+    /** Mean score across replications */
+    mean: number;
+    /** Min score observed */
+    min: number;
+    /** Provider (model under test) */
+    providerId?: string;
+    /** Score range (max - min) */
+    range: number;
+    /** Number of replications */
+    replications: number;
+    /** Standard deviation across replications */
+    stdDev: number;
+    /** Task ID */
+    taskId: string;
+}
+/** A single replicated grading judgment: one (response, rubric) pair graded N times */
+export interface ReplicatedGrading {
+    /** Feature area (derived from task description) */
+    area: string;
+    /** Which scoring dimension this rubric measures */
+    dimension: "codeCorrectness" | "docCoverage" | "taskCompletion";
+    /** Provider (model under test) that produced the original response */
+    providerId?: string;
+    /** The scores from each replication (length = N replications) */
+    scores: number[];
+    /** Which task this judgment belongs to */
+    taskId: string;
+}
+/**
+ * Analyze a set of replicated gradings and produce a full consistency report.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param gradings Array of replicated grading judgments (each graded N times)
+ * @returns GraderConsistency report with per-dimension and overall statistics
+ */
+export declare function analyzeConsistency(gradings: ReplicatedGrading[]): GraderConsistency;
+/**
+ * Compute consistency statistics for a single replicated judgment.
+ */
+export declare function analyzeJudgment(grading: ReplicatedGrading): JudgmentConsistency;
+/** Compute mean of an array of numbers */
+export declare function mean(values: number[]): number;
+/** Compute population standard deviation of an array of numbers */
+export declare function stdDev(values: number[]): number;

package/dist/pipeline/grader-consistency.js

@@ -0,0 +1,146 @@
+/**
+ * pipeline/grader-consistency.ts
+ *
+ * Pure computation module for measuring grader consistency.
+ *
+ * Takes replicated grading results (the same judgment run N times) and
+ * computes standard deviation, per-dimension consistency, and an
+ * empirically-derived noise threshold recommendation.
+ *
+ * This module has NO side effects — no file I/O, no API calls.
+ * It operates on pre-collected data only.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 1
+ */
+// ---------------------------------------------------------------------------
+// Pure computation
+// ---------------------------------------------------------------------------
+/**
+ * Analyze a set of replicated gradings and produce a full consistency report.
+ *
+ * This is the main entry point — a pure function with no side effects.
+ *
+ * @param gradings Array of replicated grading judgments (each graded N times)
+ * @returns GraderConsistency report with per-dimension and overall statistics
+ */
+export function analyzeConsistency(gradings) {
+    if (gradings.length === 0) {
+        return {
+            avgRange: 0,
+            avgStdDev: 0,
+            generatedAt: new Date().toISOString(),
+            judgments: [],
+            maxStdDev: 0,
+            perDimension: {
+                codeCorrectness: {
+                    avgRange: 0,
+                    avgStdDev: 0,
+                    judgmentCount: 0,
+                    maxStdDev: 0,
+                },
+                docCoverage: {
+                    avgRange: 0,
+                    avgStdDev: 0,
+                    judgmentCount: 0,
+                    maxStdDev: 0,
+                },
+                taskCompletion: {
+                    avgRange: 0,
+                    avgStdDev: 0,
+                    judgmentCount: 0,
+                    maxStdDev: 0,
+                },
+            },
+            recommendedThreshold: 0,
+            replications: 0,
+            totalJudgments: 0,
+        };
+    }
+    // Analyze each judgment
+    const judgments = gradings.map(analyzeJudgment);
+    // Group by dimension
+    const byDimension = {
+        codeCorrectness: judgments.filter((j) => j.dimension === "codeCorrectness"),
+        docCoverage: judgments.filter((j) => j.dimension === "docCoverage"),
+        taskCompletion: judgments.filter((j) => j.dimension === "taskCompletion"),
+    };
+    const perDimension = {
+        codeCorrectness: aggregateDimension(byDimension.codeCorrectness),
+        docCoverage: aggregateDimension(byDimension.docCoverage),
+        taskCompletion: aggregateDimension(byDimension.taskCompletion),
+    };
+    // Overall stats
+    const allStdDevs = judgments.map((j) => j.stdDev);
+    const allRanges = judgments.map((j) => j.range);
+    const replications = gradings[0]?.scores.length ?? 0;
+    // Recommended threshold: 2× the worst (highest) per-dimension avgStdDev.
+    // This means a comparison delta must exceed 2σ of the noisiest dimension
+    // to be classified as a real change rather than grader variance.
+    const maxDimensionAvgStdDev = Math.max(perDimension.taskCompletion.avgStdDev, perDimension.codeCorrectness.avgStdDev, perDimension.docCoverage.avgStdDev);
+    const recommendedThreshold = Math.ceil(maxDimensionAvgStdDev * 2);
+    // Sort judgments by stdDev descending (noisiest first)
+    const sortedJudgments = [...judgments].sort((a, b) => b.stdDev - a.stdDev);
+    return {
+        avgRange: Math.round(mean(allRanges) * 10) / 10,
+        avgStdDev: Math.round(mean(allStdDevs) * 10) / 10,
+        generatedAt: new Date().toISOString(),
+        judgments: sortedJudgments,
+        maxStdDev: Math.max(...allStdDevs),
+        perDimension,
+        recommendedThreshold,
+        replications,
+        totalJudgments: judgments.length,
+    };
+}
+/**
+ * Compute consistency statistics for a single replicated judgment.
+ */
+export function analyzeJudgment(grading) {
+    const scores = grading.scores;
+    const avg = mean(scores);
+    const sd = stdDev(scores);
+    const min = Math.min(...scores);
+    const max = Math.max(...scores);
+    return {
+        area: grading.area,
+        dimension: grading.dimension,
+        max,
+        mean: Math.round(avg * 10) / 10,
+        min,
+        range: max - min,
+        replications: scores.length,
+        stdDev: Math.round(sd * 10) / 10,
+        taskId: grading.taskId,
+        ...(grading.providerId && { providerId: grading.providerId }),
+    };
+}
+/** Compute mean of an array of numbers */
+export function mean(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((sum, v) => sum + v, 0) / values.length;
+}
+/** Compute population standard deviation of an array of numbers */
+export function stdDev(values) {
+    if (values.length < 2)
+        return 0;
+    const avg = mean(values);
+    const squaredDiffs = values.map((v) => (v - avg) ** 2);
+    return Math.sqrt(mean(squaredDiffs));
+}
+/**
+ * Aggregate judgment-level stats into a per-dimension summary.
+ */
+function aggregateDimension(judgments) {
+    if (judgments.length === 0) {
+        return { avgRange: 0, avgStdDev: 0, judgmentCount: 0, maxStdDev: 0 };
+    }
+    const stdDevs = judgments.map((j) => j.stdDev);
+    const ranges = judgments.map((j) => j.range);
+    return {
+        avgRange: Math.round(mean(ranges) * 10) / 10,
+        avgStdDev: Math.round(mean(stdDevs) * 10) / 10,
+        judgmentCount: judgments.length,
+        maxStdDev: Math.max(...stdDevs),
+    };
+}

package/dist/pipeline/grader-sensitivity-runner.d.ts

@@ -0,0 +1,40 @@
+/**
+ * pipeline/grader-sensitivity-runner.ts
+ *
+ * Orchestration module for grader sensitivity (discrimination power) testing
+ * (Phase 4).
+ *
+ * Discovers canonical reference solutions, applies programmatic degradations,
+ * grades each original/degraded pair, and calls `analyzeSensitivity()` from
+ * the pure computation module.
+ *
+ * Migrated from lib/grader-sensitivity.ts — no process.argv, no process.exit(),
+ * no module-level constants. Accepts rootDir as parameter.
+ *
+ * @see docs/exec-plans/completed/grader-reliability.md — Phase 4
+ */
+import { type GraderSensitivityResult } from "./grader-sensitivity.js";
+export interface GraderSensitivityRunnerOptions {
+    /** Filter to a specific feature area (e.g., "groq") */
+    areaFilter?: string;
+    /** Output format */
+    format?: "json" | "table";
+    /** Custom output path */
+    outputPath?: string;
+    /** Root directory of the eval package */
+    rootDir: string;
+}
+/**
+ * Format a GraderSensitivityResult as a human-readable table report.
+ * Returns a string — does NOT print to console.
+ */
+export declare function formatSensitivityReport(result: GraderSensitivityResult): string;
+/**
+ * Run grader sensitivity analysis.
+ *
+ * Discovers reference solutions, applies degradations, grades each pair
+ * with the configured grader, and analyzes discrimination power.
+ *
+ * @throws Error if no reference solutions found or no pairs generated
+ */
+export declare function runGraderSensitivity(options: GraderSensitivityRunnerOptions): Promise<GraderSensitivityResult>;